Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
singlestore-labs
GitHub Repository: singlestore-labs/singlestoredb-python
Path: blob/main/singlestoredb/management/files.py
801 views
1
#!/usr/bin/env python
2
"""SingleStore Cloud Files Management."""
3
from __future__ import annotations
4
5
import datetime
6
import glob
7
import io
8
import os
9
import re
10
from abc import ABC
11
from abc import abstractmethod
12
from typing import Any
13
from typing import cast
14
from typing import Dict
15
from typing import List
16
from typing import Literal
17
from typing import Optional
18
from typing import overload
19
from typing import Union
20
21
from .. import config
22
from ..exceptions import ManagementError
23
from .manager import Manager
24
from .utils import PathLike
25
from .utils import to_datetime
26
from .utils import vars_to_str
27
28
PERSONAL_SPACE = 'personal'
29
SHARED_SPACE = 'shared'
30
MODELS_SPACE = 'models'
31
32
33
class FilesObject(object):
34
"""
35
File / folder object.
36
37
It can belong to either a workspace stage or personal/shared space.
38
39
This object is not instantiated directly. It is used in the results
40
of various operations in ``WorkspaceGroup.stage``, ``FilesManager.personal_space``,
41
``FilesManager.shared_space`` and ``FilesManager.models_space`` methods.
42
43
"""
44
45
def __init__(
46
self,
47
name: str,
48
path: str,
49
size: int,
50
type: str,
51
format: str,
52
mimetype: str,
53
created: Optional[datetime.datetime],
54
last_modified: Optional[datetime.datetime],
55
writable: bool,
56
content: Optional[List[str]] = None,
57
):
58
#: Name of file / folder
59
self.name = name
60
61
if type == 'directory':
62
path = re.sub(r'/*$', r'', str(path)) + '/'
63
64
#: Path of file / folder
65
self.path = path
66
67
#: Size of the object (in bytes)
68
self.size = size
69
70
#: Data type: file or directory
71
self.type = type
72
73
#: Data format
74
self.format = format
75
76
#: Mime type
77
self.mimetype = mimetype
78
79
#: Datetime the object was created
80
self.created_at = created
81
82
#: Datetime the object was modified last
83
self.last_modified_at = last_modified
84
85
#: Is the object writable?
86
self.writable = writable
87
88
#: Contents of a directory
89
self.content: List[str] = content or []
90
91
self._location: Optional[FileLocation] = None
92
93
@classmethod
94
def from_dict(
95
cls,
96
obj: Dict[str, Any],
97
location: FileLocation,
98
) -> FilesObject:
99
"""
100
Construct a FilesObject from a dictionary of values.
101
102
Parameters
103
----------
104
obj : dict
105
Dictionary of values
106
location : FileLocation
107
FileLocation object to use as the parent
108
109
Returns
110
-------
111
:class:`FilesObject`
112
113
"""
114
out = cls(
115
name=obj['name'],
116
path=obj['path'],
117
size=obj['size'],
118
type=obj['type'],
119
format=obj['format'],
120
mimetype=obj['mimetype'],
121
created=to_datetime(obj.get('created')),
122
last_modified=to_datetime(obj.get('last_modified')),
123
writable=bool(obj['writable']),
124
)
125
out._location = location
126
return out
127
128
def __str__(self) -> str:
129
"""Return string representation."""
130
return vars_to_str(self)
131
132
def __repr__(self) -> str:
133
"""Return string representation."""
134
return str(self)
135
136
def open(
137
self,
138
mode: str = 'r',
139
encoding: Optional[str] = None,
140
) -> Union[io.StringIO, io.BytesIO]:
141
"""
142
Open a file path for reading or writing.
143
144
Parameters
145
----------
146
mode : str, optional
147
The read / write mode. The following modes are supported:
148
* 'r' open for reading (default)
149
* 'w' open for writing, truncating the file first
150
* 'x' create a new file and open it for writing
151
The data type can be specified by adding one of the following:
152
* 'b' binary mode
153
* 't' text mode (default)
154
encoding : str, optional
155
The string encoding to use for text
156
157
Returns
158
-------
159
FilesObjectBytesReader - 'rb' or 'b' mode
160
FilesObjectBytesWriter - 'wb' or 'xb' mode
161
FilesObjectTextReader - 'r' or 'rt' mode
162
FilesObjectTextWriter - 'w', 'x', 'wt' or 'xt' mode
163
164
"""
165
if self._location is None:
166
raise ManagementError(
167
msg='No FileLocation object is associated with this object.',
168
)
169
170
if self.is_dir():
171
raise IsADirectoryError(
172
f'directories can not be read or written: {self.path}',
173
)
174
175
return self._location.open(self.path, mode=mode, encoding=encoding)
176
177
def download(
178
self,
179
local_path: Optional[PathLike] = None,
180
*,
181
overwrite: bool = False,
182
encoding: Optional[str] = None,
183
) -> Optional[Union[bytes, str]]:
184
"""
185
Download the content of a file path.
186
187
Parameters
188
----------
189
local_path : Path or str
190
Path to local file target location
191
overwrite : bool, optional
192
Should an existing file be overwritten if it exists?
193
encoding : str, optional
194
Encoding used to convert the resulting data
195
196
Returns
197
-------
198
bytes or str or None
199
200
"""
201
if self._location is None:
202
raise ManagementError(
203
msg='No FileLocation object is associated with this object.',
204
)
205
206
return self._location.download_file(
207
self.path, local_path=local_path,
208
overwrite=overwrite, encoding=encoding,
209
)
210
211
download_file = download
212
213
def remove(self) -> None:
214
"""Delete the file."""
215
if self._location is None:
216
raise ManagementError(
217
msg='No FileLocation object is associated with this object.',
218
)
219
220
if self.type == 'directory':
221
raise IsADirectoryError(
222
f'path is a directory; use rmdir or removedirs {self.path}',
223
)
224
225
self._location.remove(self.path)
226
227
def rmdir(self) -> None:
228
"""Delete the empty directory."""
229
if self._location is None:
230
raise ManagementError(
231
msg='No FileLocation object is associated with this object.',
232
)
233
234
if self.type != 'directory':
235
raise NotADirectoryError(
236
f'path is not a directory: {self.path}',
237
)
238
239
self._location.rmdir(self.path)
240
241
def removedirs(self) -> None:
242
"""Delete the directory recursively."""
243
if self._location is None:
244
raise ManagementError(
245
msg='No FileLocation object is associated with this object.',
246
)
247
248
if self.type != 'directory':
249
raise NotADirectoryError(
250
f'path is not a directory: {self.path}',
251
)
252
253
self._location.removedirs(self.path)
254
255
def rename(self, new_path: PathLike, *, overwrite: bool = False) -> None:
256
"""
257
Move the file to a new location.
258
259
Parameters
260
----------
261
new_path : Path or str
262
The new location of the file
263
overwrite : bool, optional
264
Should path be overwritten if it already exists?
265
266
"""
267
if self._location is None:
268
raise ManagementError(
269
msg='No FileLocation object is associated with this object.',
270
)
271
out = self._location.rename(self.path, new_path, overwrite=overwrite)
272
self.name = out.name
273
self.path = out.path
274
return None
275
276
def exists(self) -> bool:
277
"""Does the file / folder exist?"""
278
if self._location is None:
279
raise ManagementError(
280
msg='No FileLocation object is associated with this object.',
281
)
282
return self._location.exists(self.path)
283
284
def is_dir(self) -> bool:
285
"""Is the object a directory?"""
286
return self.type == 'directory'
287
288
def is_file(self) -> bool:
289
"""Is the object a file?"""
290
return self.type != 'directory'
291
292
def abspath(self) -> str:
293
"""Return the full path of the object."""
294
return str(self.path)
295
296
def basename(self) -> str:
297
"""Return the basename of the object."""
298
return self.name
299
300
def dirname(self) -> str:
301
"""Return the directory name of the object."""
302
return re.sub(r'/*$', r'', os.path.dirname(re.sub(r'/*$', r'', self.path))) + '/'
303
304
def getmtime(self) -> float:
305
"""Return the last modified datetime as a UNIX timestamp."""
306
if self.last_modified_at is None:
307
return 0.0
308
return self.last_modified_at.timestamp()
309
310
def getctime(self) -> float:
311
"""Return the creation datetime as a UNIX timestamp."""
312
if self.created_at is None:
313
return 0.0
314
return self.created_at.timestamp()
315
316
317
class FilesObjectTextWriter(io.StringIO):
318
"""StringIO wrapper for writing to FileLocation."""
319
320
def __init__(self, buffer: Optional[str], location: FileLocation, path: PathLike):
321
self._location = location
322
self._path = path
323
super().__init__(buffer)
324
325
def close(self) -> None:
326
"""Write the content to the path."""
327
self._location._upload(self.getvalue(), self._path)
328
super().close()
329
330
331
class FilesObjectTextReader(io.StringIO):
332
"""StringIO wrapper for reading from FileLocation."""
333
334
335
class FilesObjectBytesWriter(io.BytesIO):
336
"""BytesIO wrapper for writing to FileLocation."""
337
338
def __init__(self, buffer: bytes, location: FileLocation, path: PathLike):
339
self._location = location
340
self._path = path
341
super().__init__(buffer)
342
343
def close(self) -> None:
344
"""Write the content to the file path."""
345
self._location._upload(self.getvalue(), self._path)
346
super().close()
347
348
349
class FilesObjectBytesReader(io.BytesIO):
350
"""BytesIO wrapper for reading from FileLocation."""
351
352
353
class FileLocation(ABC):
354
355
@abstractmethod
356
def open(
357
self,
358
path: PathLike,
359
mode: str = 'r',
360
encoding: Optional[str] = None,
361
) -> Union[io.StringIO, io.BytesIO]:
362
pass
363
364
@abstractmethod
365
def upload_file(
366
self,
367
local_path: Union[PathLike, io.IOBase],
368
path: PathLike,
369
*,
370
overwrite: bool = False,
371
) -> FilesObject:
372
pass
373
374
@abstractmethod
375
def upload_folder(
376
self,
377
local_path: PathLike,
378
path: PathLike,
379
*,
380
overwrite: bool = False,
381
recursive: bool = True,
382
include_root: bool = False,
383
ignore: Optional[Union[PathLike, List[PathLike]]] = None,
384
) -> FilesObject:
385
pass
386
387
@abstractmethod
388
def _upload(
389
self,
390
content: Union[str, bytes, io.IOBase],
391
path: PathLike,
392
*,
393
overwrite: bool = False,
394
) -> FilesObject:
395
pass
396
397
@abstractmethod
398
def mkdir(self, path: PathLike, overwrite: bool = False) -> FilesObject:
399
pass
400
401
@abstractmethod
402
def rename(
403
self,
404
old_path: PathLike,
405
new_path: PathLike,
406
*,
407
overwrite: bool = False,
408
) -> FilesObject:
409
pass
410
411
@abstractmethod
412
def info(self, path: PathLike) -> FilesObject:
413
pass
414
415
@abstractmethod
416
def exists(self, path: PathLike) -> bool:
417
pass
418
419
@abstractmethod
420
def is_dir(self, path: PathLike) -> bool:
421
pass
422
423
@abstractmethod
424
def is_file(self, path: PathLike) -> bool:
425
pass
426
427
@overload
428
def listdir(
429
self,
430
path: PathLike = '/',
431
*,
432
recursive: bool = False,
433
return_objects: Literal[True],
434
) -> List[FilesObject]:
435
pass
436
437
@overload
438
def listdir(
439
self,
440
path: PathLike = '/',
441
*,
442
recursive: bool = False,
443
return_objects: Literal[False] = False,
444
) -> List[str]:
445
pass
446
447
@abstractmethod
448
def listdir(
449
self,
450
path: PathLike = '/',
451
*,
452
recursive: bool = False,
453
return_objects: bool = False,
454
) -> Union[List[str], List[FilesObject]]:
455
pass
456
457
@abstractmethod
458
def download_file(
459
self,
460
path: PathLike,
461
local_path: Optional[PathLike] = None,
462
*,
463
overwrite: bool = False,
464
encoding: Optional[str] = None,
465
) -> Optional[Union[bytes, str]]:
466
pass
467
468
@abstractmethod
469
def download_folder(
470
self,
471
path: PathLike,
472
local_path: PathLike = '.',
473
*,
474
overwrite: bool = False,
475
) -> None:
476
pass
477
478
@abstractmethod
479
def remove(self, path: PathLike) -> None:
480
pass
481
482
@abstractmethod
483
def removedirs(self, path: PathLike) -> None:
484
pass
485
486
@abstractmethod
487
def rmdir(self, path: PathLike) -> None:
488
pass
489
490
@abstractmethod
491
def __str__(self) -> str:
492
pass
493
494
@abstractmethod
495
def __repr__(self) -> str:
496
pass
497
498
499
class FilesManager(Manager):
500
"""
501
SingleStoreDB files manager.
502
503
This class should be instantiated using :func:`singlestoredb.manage_files`.
504
505
Parameters
506
----------
507
access_token : str, optional
508
The API key or other access token for the files management API
509
version : str, optional
510
Version of the API to use
511
base_url : str, optional
512
Base URL of the files management API
513
514
See Also
515
--------
516
:func:`singlestoredb.manage_files`
517
518
"""
519
520
#: Management API version if none is specified.
521
default_version = config.get_option('management.version') or 'v1'
522
523
#: Base URL if none is specified.
524
default_base_url = config.get_option('management.base_url') \
525
or 'https://api.singlestore.com'
526
527
#: Object type
528
obj_type = 'file'
529
530
@property
531
def personal_space(self) -> FileSpace:
532
"""Return the personal file space."""
533
return FileSpace(PERSONAL_SPACE, self)
534
535
@property
536
def shared_space(self) -> FileSpace:
537
"""Return the shared file space."""
538
return FileSpace(SHARED_SPACE, self)
539
540
@property
541
def models_space(self) -> FileSpace:
542
"""Return the models file space."""
543
return FileSpace(MODELS_SPACE, self)
544
545
546
def manage_files(
547
access_token: Optional[str] = None,
548
version: Optional[str] = None,
549
base_url: Optional[str] = None,
550
*,
551
organization_id: Optional[str] = None,
552
) -> FilesManager:
553
"""
554
Retrieve a SingleStoreDB files manager.
555
556
Parameters
557
----------
558
access_token : str, optional
559
The API key or other access token for the files management API
560
version : str, optional
561
Version of the API to use
562
base_url : str, optional
563
Base URL of the files management API
564
organization_id : str, optional
565
ID of organization, if using a JWT for authentication
566
567
Returns
568
-------
569
:class:`FilesManager`
570
571
"""
572
return FilesManager(
573
access_token=access_token, base_url=base_url,
574
version=version, organization_id=organization_id,
575
)
576
577
578
class FileSpace(FileLocation):
579
"""
580
FileSpace manager.
581
582
This object is not instantiated directly.
583
It is returned by ``FilesManager.personal_space``, ``FilesManager.shared_space``
584
or ``FileManger.models_space``.
585
586
"""
587
588
def __init__(self, location: str, manager: FilesManager):
589
self._location = location
590
self._manager = manager
591
592
def open(
593
self,
594
path: PathLike,
595
mode: str = 'r',
596
encoding: Optional[str] = None,
597
) -> Union[io.StringIO, io.BytesIO]:
598
"""
599
Open a file path for reading or writing.
600
601
Parameters
602
----------
603
path : Path or str
604
The file path to read / write
605
mode : str, optional
606
The read / write mode. The following modes are supported:
607
* 'r' open for reading (default)
608
* 'w' open for writing, truncating the file first
609
* 'x' create a new file and open it for writing
610
The data type can be specified by adding one of the following:
611
* 'b' binary mode
612
* 't' text mode (default)
613
encoding : str, optional
614
The string encoding to use for text
615
616
Returns
617
-------
618
FilesObjectBytesReader - 'rb' or 'b' mode
619
FilesObjectBytesWriter - 'wb' or 'xb' mode
620
FilesObjectTextReader - 'r' or 'rt' mode
621
FilesObjectTextWriter - 'w', 'x', 'wt' or 'xt' mode
622
623
"""
624
if '+' in mode or 'a' in mode:
625
raise ManagementError(msg='modifying an existing file is not supported')
626
627
if 'w' in mode or 'x' in mode:
628
exists = self.exists(path)
629
if exists:
630
if 'x' in mode:
631
raise FileExistsError(f'file path already exists: {path}')
632
self.remove(path)
633
if 'b' in mode:
634
return FilesObjectBytesWriter(b'', self, path)
635
return FilesObjectTextWriter('', self, path)
636
637
if 'r' in mode:
638
content = self.download_file(path)
639
if isinstance(content, bytes):
640
if 'b' in mode:
641
return FilesObjectBytesReader(content)
642
encoding = 'utf-8' if encoding is None else encoding
643
return FilesObjectTextReader(content.decode(encoding))
644
645
if isinstance(content, str):
646
return FilesObjectTextReader(content)
647
648
raise ValueError(f'unrecognized file content type: {type(content)}')
649
650
raise ValueError(f'must have one of create/read/write mode specified: {mode}')
651
652
def upload_file(
653
self,
654
local_path: Union[PathLike, io.IOBase],
655
path: PathLike,
656
*,
657
overwrite: bool = False,
658
) -> FilesObject:
659
"""
660
Upload a local file.
661
662
Parameters
663
----------
664
local_path : Path or str or file-like
665
Path to the local file or an open file object
666
path : Path or str
667
Path to the file
668
overwrite : bool, optional
669
Should the ``path`` be overwritten if it exists already?
670
671
"""
672
if isinstance(local_path, io.IOBase):
673
pass
674
elif not os.path.isfile(local_path):
675
raise IsADirectoryError(f'local path is not a file: {local_path}')
676
677
if self.exists(path):
678
if not overwrite:
679
raise OSError(f'file path already exists: {path}')
680
681
self.remove(path)
682
683
if isinstance(local_path, io.IOBase):
684
return self._upload(local_path, path, overwrite=overwrite)
685
686
return self._upload(open(local_path, 'rb'), path, overwrite=overwrite)
687
688
def upload_folder(
689
self,
690
local_path: PathLike,
691
path: PathLike,
692
*,
693
overwrite: bool = False,
694
recursive: bool = True,
695
include_root: bool = False,
696
ignore: Optional[Union[PathLike, List[PathLike]]] = None,
697
) -> FilesObject:
698
"""
699
Upload a folder recursively.
700
701
Only the contents of the folder are uploaded. To include the
702
folder name itself in the target path use ``include_root=True``.
703
704
Parameters
705
----------
706
local_path : Path or str
707
Local directory to upload
708
path : Path or str
709
Path of folder to upload to
710
overwrite : bool, optional
711
If a file already exists, should it be overwritten?
712
recursive : bool, optional
713
Should nested folders be uploaded?
714
include_root : bool, optional
715
Should the local root folder itself be uploaded as the top folder?
716
ignore : Path or str or List[Path] or List[str], optional
717
Glob patterns of files to ignore, for example, '**/*.pyc` will
718
ignore all '*.pyc' files in the directory tree
719
720
"""
721
if not os.path.isdir(local_path):
722
raise NotADirectoryError(f'local path is not a directory: {local_path}')
723
724
if not path:
725
path = local_path
726
727
ignore_files = set()
728
if ignore:
729
if isinstance(ignore, list):
730
for item in ignore:
731
ignore_files.update(glob.glob(str(item), recursive=recursive))
732
else:
733
ignore_files.update(glob.glob(str(ignore), recursive=recursive))
734
735
for dir_path, _, files in os.walk(str(local_path)):
736
for fname in files:
737
if ignore_files and fname in ignore_files:
738
continue
739
740
local_file_path = os.path.join(dir_path, fname)
741
remote_path = os.path.join(
742
path,
743
local_file_path.lstrip(str(local_path)),
744
)
745
self.upload_file(
746
local_path=local_file_path,
747
path=remote_path,
748
overwrite=overwrite,
749
)
750
return self.info(path)
751
752
def _upload(
753
self,
754
content: Union[str, bytes, io.IOBase],
755
path: PathLike,
756
*,
757
overwrite: bool = False,
758
) -> FilesObject:
759
"""
760
Upload content to a file.
761
762
Parameters
763
----------
764
content : str or bytes or file-like
765
Content to upload
766
path : Path or str
767
Path to the file
768
overwrite : bool, optional
769
Should the ``path`` be overwritten if it exists already?
770
771
"""
772
if self.exists(path):
773
if not overwrite:
774
raise OSError(f'file path already exists: {path}')
775
self.remove(path)
776
777
self._manager._put(
778
f'files/fs/{self._location}/{path}',
779
files={'file': content},
780
headers={'Content-Type': None},
781
)
782
783
return self.info(path)
784
785
def mkdir(self, path: PathLike, overwrite: bool = False) -> FilesObject:
786
"""
787
Make a directory in the file space.
788
789
Parameters
790
----------
791
path : Path or str
792
Path of the folder to create
793
overwrite : bool, optional
794
Should the file path be overwritten if it exists already?
795
796
Returns
797
-------
798
FilesObject
799
800
"""
801
raise ManagementError(
802
msg='Operation not supported: directories are currently not allowed '
803
'in Files API',
804
)
805
806
mkdirs = mkdir
807
808
def rename(
809
self,
810
old_path: PathLike,
811
new_path: PathLike,
812
*,
813
overwrite: bool = False,
814
) -> FilesObject:
815
"""
816
Move the file to a new location.
817
818
Parameters
819
-----------
820
old_path : Path or str
821
Original location of the path
822
new_path : Path or str
823
New location of the path
824
overwrite : bool, optional
825
Should the ``new_path`` be overwritten if it exists already?
826
827
"""
828
if not self.exists(old_path):
829
raise OSError(f'file path does not exist: {old_path}')
830
831
if str(old_path).endswith('/') or str(new_path).endswith('/'):
832
raise ManagementError(
833
msg='Operation not supported: directories are currently not allowed '
834
'in Files API',
835
)
836
837
if self.exists(new_path):
838
if not overwrite:
839
raise OSError(f'file path already exists: {new_path}')
840
841
self.remove(new_path)
842
843
self._manager._patch(
844
f'files/fs/{self._location}/{old_path}',
845
json=dict(newPath=new_path),
846
)
847
848
return self.info(new_path)
849
850
def info(self, path: PathLike) -> FilesObject:
851
"""
852
Return information about a file location.
853
854
Parameters
855
----------
856
path : Path or str
857
Path to the file
858
859
Returns
860
-------
861
FilesObject
862
863
"""
864
res = self._manager._get(
865
re.sub(r'/+$', r'/', f'files/fs/{self._location}/{path}'),
866
params=dict(metadata=1),
867
).json()
868
869
return FilesObject.from_dict(res, self)
870
871
def exists(self, path: PathLike) -> bool:
872
"""
873
Does the given file path exist?
874
875
Parameters
876
----------
877
path : Path or str
878
Path to file object
879
880
Returns
881
-------
882
bool
883
884
"""
885
try:
886
self.info(path)
887
return True
888
except ManagementError as exc:
889
if exc.errno == 404:
890
return False
891
raise
892
893
def is_dir(self, path: PathLike) -> bool:
894
"""
895
Is the given file path a directory?
896
897
Parameters
898
----------
899
path : Path or str
900
Path to file object
901
902
Returns
903
-------
904
bool
905
906
"""
907
try:
908
return self.info(path).type == 'directory'
909
except ManagementError as exc:
910
if exc.errno == 404:
911
return False
912
raise
913
914
def is_file(self, path: PathLike) -> bool:
915
"""
916
Is the given file path a file?
917
918
Parameters
919
----------
920
path : Path or str
921
Path to file object
922
923
Returns
924
-------
925
bool
926
927
"""
928
try:
929
return self.info(path).type != 'directory'
930
except ManagementError as exc:
931
if exc.errno == 404:
932
return False
933
raise
934
935
def _listdir(
936
self, path: PathLike, *,
937
recursive: bool = False,
938
return_objects: bool = False,
939
) -> List[Union[str, FilesObject]]:
940
"""
941
Return the names (or FilesObject instances) of files in a directory.
942
943
Parameters
944
----------
945
path : Path or str
946
Path to the folder
947
recursive : bool, optional
948
Should folders be listed recursively?
949
return_objects : bool, optional
950
If True, return list of FilesObject instances. Otherwise just paths.
951
"""
952
res = self._manager._get(
953
f'files/fs/{self._location}/{path}',
954
).json()
955
956
if recursive:
957
out: List[Union[str, FilesObject]] = []
958
for item in res.get('content') or []:
959
if return_objects:
960
out.append(FilesObject.from_dict(item, self))
961
else:
962
out.append(item['path'])
963
if item['type'] == 'directory':
964
out.extend(
965
self._listdir(
966
item['path'],
967
recursive=recursive,
968
return_objects=return_objects,
969
),
970
)
971
return out
972
973
if return_objects:
974
return [
975
FilesObject.from_dict(x, self)
976
for x in (res.get('content') or [])
977
]
978
return [x['path'] for x in (res.get('content') or [])]
979
980
@overload
981
def listdir(
982
self,
983
path: PathLike = '/',
984
*,
985
recursive: bool = False,
986
return_objects: Literal[True],
987
) -> List[FilesObject]:
988
...
989
990
@overload
991
def listdir(
992
self,
993
path: PathLike = '/',
994
*,
995
recursive: bool = False,
996
return_objects: Literal[False] = False,
997
) -> List[str]:
998
...
999
1000
def listdir(
1001
self,
1002
path: PathLike = '/',
1003
*,
1004
recursive: bool = False,
1005
return_objects: bool = False,
1006
) -> Union[List[str], List[FilesObject]]:
1007
"""
1008
List the files / folders at the given path.
1009
1010
Parameters
1011
----------
1012
path : Path or str, optional
1013
Path to the file location
1014
1015
return_objects : bool, optional
1016
If True, return list of FilesObject instances. Otherwise just paths.
1017
1018
Returns
1019
-------
1020
List[str] or List[FilesObject]
1021
1022
"""
1023
path = re.sub(r'^(\./|/)+', r'', str(path))
1024
path = re.sub(r'/+$', r'', path) + '/'
1025
1026
# Validate via listing GET; if response lacks 'content', it's not a directory
1027
try:
1028
out = self._listdir(path, recursive=recursive, return_objects=return_objects)
1029
except (ManagementError, KeyError) as exc:
1030
# If the path doesn't exist or isn't a directory, _listdir will fail
1031
raise NotADirectoryError(f'path is not a directory: {path}') from exc
1032
1033
if path != '/':
1034
path_n = len(path.split('/')) - 1
1035
if return_objects:
1036
result: List[FilesObject] = []
1037
for item in out:
1038
if isinstance(item, FilesObject):
1039
rel = '/'.join(item.path.split('/')[path_n:])
1040
item.path = rel
1041
result.append(item)
1042
return result
1043
return ['/'.join(str(x).split('/')[path_n:]) for x in out]
1044
1045
# _listdir guarantees homogeneous type based on return_objects
1046
if return_objects:
1047
return cast(List[FilesObject], out)
1048
return cast(List[str], out)
1049
1050
def download_file(
1051
self,
1052
path: PathLike,
1053
local_path: Optional[PathLike] = None,
1054
*,
1055
overwrite: bool = False,
1056
encoding: Optional[str] = None,
1057
) -> Optional[Union[bytes, str]]:
1058
"""
1059
Download the content of a file path.
1060
1061
Parameters
1062
----------
1063
path : Path or str
1064
Path to the file
1065
local_path : Path or str
1066
Path to local file target location
1067
overwrite : bool, optional
1068
Should an existing file be overwritten if it exists?
1069
encoding : str, optional
1070
Encoding used to convert the resulting data
1071
1072
Returns
1073
-------
1074
bytes or str - ``local_path`` is None
1075
None - ``local_path`` is a Path or str
1076
1077
"""
1078
return self._download_file(
1079
path,
1080
local_path=local_path,
1081
overwrite=overwrite,
1082
encoding=encoding,
1083
_skip_dir_check=False,
1084
)
1085
1086
def _download_file(
1087
self,
1088
path: PathLike,
1089
local_path: Optional[PathLike] = None,
1090
*,
1091
overwrite: bool = False,
1092
encoding: Optional[str] = None,
1093
_skip_dir_check: bool = False,
1094
) -> Optional[Union[bytes, str]]:
1095
"""
1096
Internal method to download the content of a file path.
1097
1098
Parameters
1099
----------
1100
path : Path or str
1101
Path to the file
1102
local_path : Path or str
1103
Path to local file target location
1104
overwrite : bool, optional
1105
Should an existing file be overwritten if it exists?
1106
encoding : str, optional
1107
Encoding used to convert the resulting data
1108
_skip_dir_check : bool, optional
1109
Skip the directory check (internal use only)
1110
1111
Returns
1112
-------
1113
bytes or str - ``local_path`` is None
1114
None - ``local_path`` is a Path or str
1115
1116
"""
1117
if local_path is not None and not overwrite and os.path.exists(local_path):
1118
raise OSError('target file already exists; use overwrite=True to replace')
1119
if not _skip_dir_check and self.is_dir(path):
1120
raise IsADirectoryError(f'file path is a directory: {path}')
1121
1122
out = self._manager._get(
1123
f'files/fs/{self._location}/{path}',
1124
).content
1125
1126
if local_path is not None:
1127
with open(local_path, 'wb') as outfile:
1128
outfile.write(out)
1129
return None
1130
1131
if encoding:
1132
return out.decode(encoding)
1133
1134
return out
1135
1136
def download_folder(
1137
self,
1138
path: PathLike,
1139
local_path: PathLike = '.',
1140
*,
1141
overwrite: bool = False,
1142
) -> None:
1143
"""
1144
Download a FileSpace folder to a local directory.
1145
1146
Parameters
1147
----------
1148
path : Path or str
1149
Directory path
1150
local_path : Path or str
1151
Path to local directory target location
1152
overwrite : bool, optional
1153
Should an existing directory / files be overwritten if they exist?
1154
1155
"""
1156
1157
if local_path is not None and not overwrite and os.path.exists(local_path):
1158
raise OSError('target path already exists; use overwrite=True to replace')
1159
1160
# listdir validates directory; no extra info call needed
1161
entries = self.listdir(path, recursive=True, return_objects=True)
1162
for entry in entries:
1163
# Each entry is a FilesObject with path relative to root and type
1164
if not isinstance(entry, FilesObject): # defensive: skip unexpected
1165
continue
1166
rel_path = entry.path
1167
if entry.type == 'directory':
1168
# Ensure local directory exists; no remote call needed
1169
target_dir = os.path.normpath(os.path.join(local_path, rel_path))
1170
os.makedirs(target_dir, exist_ok=True)
1171
continue
1172
remote_path = os.path.join(path, rel_path)
1173
target_file = os.path.normpath(
1174
os.path.join(local_path, rel_path),
1175
)
1176
os.makedirs(os.path.dirname(target_file), exist_ok=True)
1177
self._download_file(
1178
remote_path, target_file,
1179
overwrite=overwrite, _skip_dir_check=True,
1180
)
1181
1182
def remove(self, path: PathLike) -> None:
1183
"""
1184
Delete a file location.
1185
1186
Parameters
1187
----------
1188
path : Path or str
1189
Path to the location
1190
1191
"""
1192
if self.is_dir(path):
1193
raise IsADirectoryError('file path is a directory')
1194
1195
self._manager._delete(f'files/fs/{self._location}/{path}')
1196
1197
def removedirs(self, path: PathLike) -> None:
1198
"""
1199
Delete a folder recursively.
1200
1201
Parameters
1202
----------
1203
path : Path or str
1204
Path to the file location
1205
1206
"""
1207
if not self.is_dir(path):
1208
raise NotADirectoryError('path is not a directory')
1209
1210
self._manager._delete(f'files/fs/{self._location}/{path}')
1211
1212
def rmdir(self, path: PathLike) -> None:
1213
"""
1214
Delete a folder.
1215
1216
Parameters
1217
----------
1218
path : Path or str
1219
Path to the file location
1220
1221
"""
1222
raise ManagementError(
1223
msg='Operation not supported: directories are currently not allowed '
1224
'in Files API',
1225
)
1226
1227
def __str__(self) -> str:
1228
"""Return string representation."""
1229
return vars_to_str(self)
1230
1231
def __repr__(self) -> str:
1232
"""Return string representation."""
1233
return str(self)
1234
1235