azure_utils.blob_storage

View Source

  1import os, pathlib
  2from azure.storage.blob import ContainerClient
  3
  4from azure_utils.utils.blob_storage_utils import AZBlob, MetaBlob, validate_path
  5
  6
  7class AZContainer:
  8    def __init__(
  9        self,
 10        account_url: str,
 11        account_key: str,
 12        container_name: str,
 13        show_progress: bool = False,
 14    ):
 15        """
 16        Function:
 17
 18        - Creates an AZContainer object
 19
 20        Parameters:
 21
 22        - `account_url` (str): The account url
 23        - `account_key` (str): The account key
 24        - `container_name` (str): The container name
 25        - `show_progress` (bool): Whether show progress (log files) when uploading and downloading files
 26
 27        Returns:
 28
 29        - The AZContainer object
 30
 31        Example:
 32
 33        ```python
 34        from azure_utils.blob_storage import AZContainer
 35        mycontainer = AZContainer(
 36            account_url="my_account_url",
 37            account_key="my_account_key",
 38            container_name="my_container"
 39        )
 40        print(mycontainer.list_files(remote_folderpath='/path/to/folder/')
 41        # This will print a list of all files in the remote folder `/path/to/folder/`
 42        ```
 43        """
 44        self.show_progress = show_progress
 45        self.client = ContainerClient(
 46            account_url=account_url,
 47            container_name=container_name,
 48            credential=account_key,
 49        )
 50
 51    def list_files(self, remote_folderpath: str) -> list:
 52        """
 53        Function:
 54
 55        - Lists all files in a remote folder
 56
 57        Parameters:
 58
 59        - `remote_folderpath` (str): The path to the remote folder
 60
 61        Returns:
 62
 63        - A list of all files (as strings) in the remote folder (including subfolders)
 64
 65        Example:
 66
 67        ```python
 68        from azure_utils.blob_storage import AZContainer
 69        mycontainer = AZContainer(
 70            account_url="my_account_url",
 71            account_key="my_account_key",
 72            container_name="my_container"
 73        )
 74        print(mycontainer.list_files(remote_folderpath='/path/to/folder/')
 75        # This will print a list of all files in the remote folder `/path/to/folder/`
 76        ```
 77        """
 78        validate_path(path=remote_folderpath, is_remote=True, is_folder=True)
 79        return [
 80            i.name
 81            for i in self.client.list_blobs(name_starts_with=remote_folderpath)
 82            if i.size > 0
 83        ]
 84
 85    def upload_file(
 86        self, remote_filepath: str, local_filepath: str, overwrite: bool = False
 87    ) -> None:
 88        """
 89        Function:
 90
 91        - Uploads a file from local to remote
 92
 93        Parameters:
 94
 95        - `remote_filepath` (str): The path to the remote file
 96        - `local_filepath` (str): The path to the local file
 97        - `overwrite` (bool): Whether to overwrite the remote file if it already exists
 98            - Optional: Defaults to `False`
 99
100        Returns:
101
102        - `None`
103
104        Example:
105
106        ```python
107        from azure_utils.blob_storage import AZContainer
108        mycontainer = AZContainer(
109            account_url="my_account_url",
110            account_key="my_account_key",
111            container_name="my_container"
112        )
113        mycontainer.upload_file(
114            remote_filepath='/path/to/file.txt',
115            local_filepath='/path/to/file.txt',
116            overwrite=True
117        )
118        # This will upload the local file `/path/to/file.txt` to the remote file `/path/to/file.txt`
119        ```
120        """
121        validate_path(path=local_filepath, is_remote=False, is_folder=False)
122        validate_path(path=remote_filepath, is_remote=True, is_folder=False)
123        if self.show_progress:
124            print(f"Uploading {local_filepath} to {remote_filepath}")
125        with open(local_filepath, "rb") as data:
126            blob = MetaBlob(
127                blob_client=self.client.upload_blob(
128                    name=remote_filepath, data=data, overwrite=overwrite
129                ),
130                filepath=local_filepath,
131            )
132        blob.update_meta()
133
134    def download_file(
135        self,
136        remote_filepath: str,
137        local_filepath: str,
138        overwrite: bool = False,
139        smart_sync: bool = False,
140    ) -> None:
141        """
142        Function:
143
144        - Downloads a file from remote to local
145
146        Parameters:
147
148        - `remote_filepath` (str): The path to the remote file
149        - `local_filepath` (str): The path to the local file
150        - `overwrite` (bool): Whether to overwrite the local file if it already exists
151            - Optional: Defaults to `False`
152        - `smart_sync` (bool): Whether to skip downloading if the remote etag and md5 hash match the local meta file
153            - Optional: Defaults to `False`
154
155        Returns:
156
157        - `None`
158
159        Example:
160
161        ```python
162        from azure_utils.blob_storage import AZContainer
163        mycontainer = AZContainer(
164            account_url="my_account_url",
165            account_key="my_account_key",
166            container_name="my_container"
167        )
168        mycontainer.download_file(
169            remote_filepath='/path/to/file.txt',
170            local_filepath='/path/to/file.txt',
171            overwrite=True,
172            smart_sync=True
173        )
174        # This will download the remote file `/path/to/file.txt` to the local file `/path/to/file.txt`
175        ```
176        """
177        validate_path(path=local_filepath, is_remote=False, is_folder=False)
178        validate_path(path=remote_filepath, is_remote=True, is_folder=False)
179        if self.show_progress:
180            print(f"Downloading {remote_filepath} to {local_filepath}")
181        blob = MetaBlob(
182            blob_client=self.client.get_blob_client(blob=remote_filepath),
183            filepath=local_filepath,
184            smart_sync=smart_sync,
185            overwrite=overwrite,
186        )
187        blob.download()
188
189    def delete_file(self, remote_filepath: str) -> None:
190        """
191        Function:
192
193        - Deletes a file from remote
194
195        Parameters:
196
197        - `remote_filepath` (str): The path to the remote file
198
199        Returns:
200
201        - `None`
202
203        Example:
204        ```python
205        from azure_utils.blob_storage import AZContainer
206        mycontainer = AZContainer(
207            account_url="my_account_url",
208            account_key="my_account_key",
209            container_name="my_container"
210        )
211        mycontainer.delete_file(remote_filepath='/path/to/file.txt')
212        ```
213        """
214        validate_path(path=remote_filepath, is_remote=True, is_folder=False)
215        if self.show_progress:
216            print(f"Deleting {remote_filepath}")
217        blob = AZBlob(
218            blob_client=self.client.get_blob_client(blob=remote_filepath)
219        )
220        blob.delete()
221
222    def sync_to_local(
223        self,
224        remote_folderpath: str,
225        local_folderpath: str,
226        overwrite: bool = False,
227        smart_sync: bool = False,
228        removal: bool = False,
229    ) -> None:
230        """
231        Function:
232
233        - Sync all files from a remote folder to a local folder
234
235        Parameters:
236
237        - `remote_folderpath` (str): The path to the remote folder
238        - `local_folderpath` (str): The path to the local folder
239        - `overwrite` (bool): Whether to overwrite the local file if it already exists
240            - Optional: Defaults to `False`
241        - `smart_sync` (bool): Whether to skip downloading if the remote etag and md5 hash match the local meta file
242            - Optional: Defaults to `False`
243        - `removal` (bool): Whether to remove local files that do not exist in the remote folder
244            - Optional: Defaults to `False`
245
246        Returns:
247
248        - `None`
249
250        Example:
251
252        ```python
253        from azure_utils.blob_storage import AZContainer
254        mycontainer = AZContainer(
255            account_url="my_account_url",
256            account_key="my_account_key",
257            container_name="my_container"
258        )
259        mycontainer.sync_to_local(
260            remote_folderpath='/path/to/folder/',
261            local_folderpath='/path/to/folder/',
262            overwrite=True,
263            smart_sync=True,
264            removal=True
265        )
266        # This will sync the remote folder `/path/to/folder/` to the local folder `/path/to/folder/`
267        # It will also remove any local files that do not exist in the remote folder
268        ```
269        """
270        validate_path(path=local_folderpath, is_remote=False, is_folder=True)
271        validate_path(path=remote_folderpath, is_remote=True, is_folder=True)
272        blobs = [
273            i
274            for i in self.client.list_blobs(name_starts_with=remote_folderpath)
275            if i.size > 0
276        ]
277        for blobProperty in blobs:
278            blob = MetaBlob(
279                blob_client=None,
280                filepath=local_folderpath
281                + blobProperty.name.replace(remote_folderpath, "/", 1),
282                smart_sync=smart_sync,
283                remote_etag=blobProperty.etag,
284                overwrite=overwrite,
285            )
286            if blob.block_download:
287                continue
288            blob.blob_client = self.client.get_blob_client(
289                blob=blobProperty.name
290            )
291            blob.download()
292        if removal:
293            posix_folderpath = str(pathlib.PurePosixPath(local_folderpath))
294            if posix_folderpath == "/":
295                raise Exception("Cannot do removal in the root folder")
296            storage_filepaths = set(
297                [i.name.replace(remote_folderpath, "") for i in blobs]
298            )
299            for local_filepath in [
300                str(i) for i in pathlib.Path(local_folderpath).glob("**/*")
301            ]:
302                posix_filepath = str(pathlib.PurePosixPath(local_filepath))
303                if (
304                    posix_filepath.replace(posix_folderpath, "").replace(
305                        ".meta.", ""
306                    )
307                    not in storage_filepaths
308                ):
309                    os.remove(local_filepath)
310
311    def sync_to_remote(
312        self,
313        remote_folderpath: str,
314        local_folderpath: str,
315        overwrite: bool = False,
316        omissions: list = [".meta."],
317    ) -> None:
318        """
319        Function:
320
321        - Sync all files from a local folder to a remote folder
322
323        Parameters:
324
325        - `remote_folderpath` (str): The path to the remote folder
326        - `local_folderpath` (str): The path to the local folder
327        - `overwrite` (bool): Whether to overwrite the remote file if it already exists
328            - Optional: Defaults to `False`
329        - `omissions` (list): A list of strings to omit from the upload
330            - Optional: Defaults to `['.meta.']`
331
332        Returns:
333
334        - `None`
335
336        Example:
337        ```python
338        from azure_utils.blob_storage import AZContainer
339        mycontainer = AZContainer(
340            account_url="my_account_url",
341            account_key="my_account_key",
342            container_name="my_container"
343        )
344        mycontainer.sync_to_remote(
345            remote_folderpath='/path/to/folder/',
346            local_folderpath='/path/to/folder/',
347        )
348        # This will sync the local folder `/path/to/folder/` to the remote folder `/path/to/folder/`
349        ```
350        """
351        validate_path(path=local_folderpath, is_remote=False, is_folder=True)
352        validate_path(path=remote_folderpath, is_remote=True, is_folder=True)
353        local_filepaths = [
354            str(i)
355            for i in pathlib.Path(local_folderpath).glob("**/*")
356            if i.is_file()
357        ]
358        local_filepaths = [
359            i for i in local_filepaths if not any([j in i for j in omissions])
360        ]
361        for local_filepath in local_filepaths:
362            self.upload_file(
363                remote_filepath=remote_folderpath
364                + local_filepath.replace(local_folderpath, ""),
365                local_filepath=local_filepath,
366                overwrite=overwrite,
367            )
368
369    def delete_folder(self, remote_folderpath: str) -> None:
370        """
371        Function:
372
373        - Deletes all files in a remote folder
374
375        Parameters:
376
377        - `remote_folderpath` (str): The path to the remote folder
378
379        Returns:
380
381        - `None`
382
383        Example:
384
385        ```python
386        from azure_utils.blob_storage import AZContainer
387        mycontainer = AZContainer(
388            account_url="my_account_url",
389            account_key="my_account_key",
390            container_name="my_container"
391        )
392        mycontainer.delete_folder(remote_folderpath='/path/to/folder/')
393        ```
394        """
395        validate_path(path=remote_folderpath, is_remote=True, is_folder=True)
396        storage_filenames = self.client.list_blob_names(
397            name_starts_with=remote_folderpath
398        )
399        for filename in storage_filenames:
400            blob = AZBlob(
401                blob_client=self.client.get_blob_client(blob=filename)
402            )
403            blob.delete()
404
405    def clear_local_meta(
406        self, local_folderpath: str, includsions: list = [".meta."]
407    ) -> None:
408        """
409        Function:
410
411        - Deletes all meta files in a local folder
412
413        Parameters:
414
415        - `local_folderpath` (str): The path to the local folder
416        - `includsions` (list): A list of strings to include in the deletion
417            - Optional: Defaults to `['.meta.']`
418
419        Returns:
420
421        - `None`
422
423        Example:
424
425        ```python
426        from azure_utils.blob_storage import AZContainer
427        mycontainer = AZContainer(
428            account_url="my_account_url",
429            account_key="my_account_key",
430            container_name="my_container"
431        )
432        mycontainer.clear_local_meta(local_folderpath='/path/to/folder/')
433        ```
434        """
435        validate_path(path=local_folderpath, is_remote=False, is_folder=True)
436        local_filepaths = [
437            str(i) for i in pathlib.Path(local_folderpath).glob("**/*")
438        ]
439        for filepath in [
440            i for i in local_filepaths if any([j in i for j in includsions])
441        ]:
442            os.remove(filepath)

class AZContainer: View Source

  8class AZContainer:
  9    def __init__(
 10        self,
 11        account_url: str,
 12        account_key: str,
 13        container_name: str,
 14        show_progress: bool = False,
 15    ):
 16        """
 17        Function:
 18
 19        - Creates an AZContainer object
 20
 21        Parameters:
 22
 23        - `account_url` (str): The account url
 24        - `account_key` (str): The account key
 25        - `container_name` (str): The container name
 26        - `show_progress` (bool): Whether show progress (log files) when uploading and downloading files
 27
 28        Returns:
 29
 30        - The AZContainer object
 31
 32        Example:
 33
 34        ```python
 35        from azure_utils.blob_storage import AZContainer
 36        mycontainer = AZContainer(
 37            account_url="my_account_url",
 38            account_key="my_account_key",
 39            container_name="my_container"
 40        )
 41        print(mycontainer.list_files(remote_folderpath='/path/to/folder/')
 42        # This will print a list of all files in the remote folder `/path/to/folder/`
 43        ```
 44        """
 45        self.show_progress = show_progress
 46        self.client = ContainerClient(
 47            account_url=account_url,
 48            container_name=container_name,
 49            credential=account_key,
 50        )
 51
 52    def list_files(self, remote_folderpath: str) -> list:
 53        """
 54        Function:
 55
 56        - Lists all files in a remote folder
 57
 58        Parameters:
 59
 60        - `remote_folderpath` (str): The path to the remote folder
 61
 62        Returns:
 63
 64        - A list of all files (as strings) in the remote folder (including subfolders)
 65
 66        Example:
 67
 68        ```python
 69        from azure_utils.blob_storage import AZContainer
 70        mycontainer = AZContainer(
 71            account_url="my_account_url",
 72            account_key="my_account_key",
 73            container_name="my_container"
 74        )
 75        print(mycontainer.list_files(remote_folderpath='/path/to/folder/')
 76        # This will print a list of all files in the remote folder `/path/to/folder/`
 77        ```
 78        """
 79        validate_path(path=remote_folderpath, is_remote=True, is_folder=True)
 80        return [
 81            i.name
 82            for i in self.client.list_blobs(name_starts_with=remote_folderpath)
 83            if i.size > 0
 84        ]
 85
 86    def upload_file(
 87        self, remote_filepath: str, local_filepath: str, overwrite: bool = False
 88    ) -> None:
 89        """
 90        Function:
 91
 92        - Uploads a file from local to remote
 93
 94        Parameters:
 95
 96        - `remote_filepath` (str): The path to the remote file
 97        - `local_filepath` (str): The path to the local file
 98        - `overwrite` (bool): Whether to overwrite the remote file if it already exists
 99            - Optional: Defaults to `False`
100
101        Returns:
102
103        - `None`
104
105        Example:
106
107        ```python
108        from azure_utils.blob_storage import AZContainer
109        mycontainer = AZContainer(
110            account_url="my_account_url",
111            account_key="my_account_key",
112            container_name="my_container"
113        )
114        mycontainer.upload_file(
115            remote_filepath='/path/to/file.txt',
116            local_filepath='/path/to/file.txt',
117            overwrite=True
118        )
119        # This will upload the local file `/path/to/file.txt` to the remote file `/path/to/file.txt`
120        ```
121        """
122        validate_path(path=local_filepath, is_remote=False, is_folder=False)
123        validate_path(path=remote_filepath, is_remote=True, is_folder=False)
124        if self.show_progress:
125            print(f"Uploading {local_filepath} to {remote_filepath}")
126        with open(local_filepath, "rb") as data:
127            blob = MetaBlob(
128                blob_client=self.client.upload_blob(
129                    name=remote_filepath, data=data, overwrite=overwrite
130                ),
131                filepath=local_filepath,
132            )
133        blob.update_meta()
134
135    def download_file(
136        self,
137        remote_filepath: str,
138        local_filepath: str,
139        overwrite: bool = False,
140        smart_sync: bool = False,
141    ) -> None:
142        """
143        Function:
144
145        - Downloads a file from remote to local
146
147        Parameters:
148
149        - `remote_filepath` (str): The path to the remote file
150        - `local_filepath` (str): The path to the local file
151        - `overwrite` (bool): Whether to overwrite the local file if it already exists
152            - Optional: Defaults to `False`
153        - `smart_sync` (bool): Whether to skip downloading if the remote etag and md5 hash match the local meta file
154            - Optional: Defaults to `False`
155
156        Returns:
157
158        - `None`
159
160        Example:
161
162        ```python
163        from azure_utils.blob_storage import AZContainer
164        mycontainer = AZContainer(
165            account_url="my_account_url",
166            account_key="my_account_key",
167            container_name="my_container"
168        )
169        mycontainer.download_file(
170            remote_filepath='/path/to/file.txt',
171            local_filepath='/path/to/file.txt',
172            overwrite=True,
173            smart_sync=True
174        )
175        # This will download the remote file `/path/to/file.txt` to the local file `/path/to/file.txt`
176        ```
177        """
178        validate_path(path=local_filepath, is_remote=False, is_folder=False)
179        validate_path(path=remote_filepath, is_remote=True, is_folder=False)
180        if self.show_progress:
181            print(f"Downloading {remote_filepath} to {local_filepath}")
182        blob = MetaBlob(
183            blob_client=self.client.get_blob_client(blob=remote_filepath),
184            filepath=local_filepath,
185            smart_sync=smart_sync,
186            overwrite=overwrite,
187        )
188        blob.download()
189
190    def delete_file(self, remote_filepath: str) -> None:
191        """
192        Function:
193
194        - Deletes a file from remote
195
196        Parameters:
197
198        - `remote_filepath` (str): The path to the remote file
199
200        Returns:
201
202        - `None`
203
204        Example:
205        ```python
206        from azure_utils.blob_storage import AZContainer
207        mycontainer = AZContainer(
208            account_url="my_account_url",
209            account_key="my_account_key",
210            container_name="my_container"
211        )
212        mycontainer.delete_file(remote_filepath='/path/to/file.txt')
213        ```
214        """
215        validate_path(path=remote_filepath, is_remote=True, is_folder=False)
216        if self.show_progress:
217            print(f"Deleting {remote_filepath}")
218        blob = AZBlob(
219            blob_client=self.client.get_blob_client(blob=remote_filepath)
220        )
221        blob.delete()
222
223    def sync_to_local(
224        self,
225        remote_folderpath: str,
226        local_folderpath: str,
227        overwrite: bool = False,
228        smart_sync: bool = False,
229        removal: bool = False,
230    ) -> None:
231        """
232        Function:
233
234        - Sync all files from a remote folder to a local folder
235
236        Parameters:
237
238        - `remote_folderpath` (str): The path to the remote folder
239        - `local_folderpath` (str): The path to the local folder
240        - `overwrite` (bool): Whether to overwrite the local file if it already exists
241            - Optional: Defaults to `False`
242        - `smart_sync` (bool): Whether to skip downloading if the remote etag and md5 hash match the local meta file
243            - Optional: Defaults to `False`
244        - `removal` (bool): Whether to remove local files that do not exist in the remote folder
245            - Optional: Defaults to `False`
246
247        Returns:
248
249        - `None`
250
251        Example:
252
253        ```python
254        from azure_utils.blob_storage import AZContainer
255        mycontainer = AZContainer(
256            account_url="my_account_url",
257            account_key="my_account_key",
258            container_name="my_container"
259        )
260        mycontainer.sync_to_local(
261            remote_folderpath='/path/to/folder/',
262            local_folderpath='/path/to/folder/',
263            overwrite=True,
264            smart_sync=True,
265            removal=True
266        )
267        # This will sync the remote folder `/path/to/folder/` to the local folder `/path/to/folder/`
268        # It will also remove any local files that do not exist in the remote folder
269        ```
270        """
271        validate_path(path=local_folderpath, is_remote=False, is_folder=True)
272        validate_path(path=remote_folderpath, is_remote=True, is_folder=True)
273        blobs = [
274            i
275            for i in self.client.list_blobs(name_starts_with=remote_folderpath)
276            if i.size > 0
277        ]
278        for blobProperty in blobs:
279            blob = MetaBlob(
280                blob_client=None,
281                filepath=local_folderpath
282                + blobProperty.name.replace(remote_folderpath, "/", 1),
283                smart_sync=smart_sync,
284                remote_etag=blobProperty.etag,
285                overwrite=overwrite,
286            )
287            if blob.block_download:
288                continue
289            blob.blob_client = self.client.get_blob_client(
290                blob=blobProperty.name
291            )
292            blob.download()
293        if removal:
294            posix_folderpath = str(pathlib.PurePosixPath(local_folderpath))
295            if posix_folderpath == "/":
296                raise Exception("Cannot do removal in the root folder")
297            storage_filepaths = set(
298                [i.name.replace(remote_folderpath, "") for i in blobs]
299            )
300            for local_filepath in [
301                str(i) for i in pathlib.Path(local_folderpath).glob("**/*")
302            ]:
303                posix_filepath = str(pathlib.PurePosixPath(local_filepath))
304                if (
305                    posix_filepath.replace(posix_folderpath, "").replace(
306                        ".meta.", ""
307                    )
308                    not in storage_filepaths
309                ):
310                    os.remove(local_filepath)
311
312    def sync_to_remote(
313        self,
314        remote_folderpath: str,
315        local_folderpath: str,
316        overwrite: bool = False,
317        omissions: list = [".meta."],
318    ) -> None:
319        """
320        Function:
321
322        - Sync all files from a local folder to a remote folder
323
324        Parameters:
325
326        - `remote_folderpath` (str): The path to the remote folder
327        - `local_folderpath` (str): The path to the local folder
328        - `overwrite` (bool): Whether to overwrite the remote file if it already exists
329            - Optional: Defaults to `False`
330        - `omissions` (list): A list of strings to omit from the upload
331            - Optional: Defaults to `['.meta.']`
332
333        Returns:
334
335        - `None`
336
337        Example:
338        ```python
339        from azure_utils.blob_storage import AZContainer
340        mycontainer = AZContainer(
341            account_url="my_account_url",
342            account_key="my_account_key",
343            container_name="my_container"
344        )
345        mycontainer.sync_to_remote(
346            remote_folderpath='/path/to/folder/',
347            local_folderpath='/path/to/folder/',
348        )
349        # This will sync the local folder `/path/to/folder/` to the remote folder `/path/to/folder/`
350        ```
351        """
352        validate_path(path=local_folderpath, is_remote=False, is_folder=True)
353        validate_path(path=remote_folderpath, is_remote=True, is_folder=True)
354        local_filepaths = [
355            str(i)
356            for i in pathlib.Path(local_folderpath).glob("**/*")
357            if i.is_file()
358        ]
359        local_filepaths = [
360            i for i in local_filepaths if not any([j in i for j in omissions])
361        ]
362        for local_filepath in local_filepaths:
363            self.upload_file(
364                remote_filepath=remote_folderpath
365                + local_filepath.replace(local_folderpath, ""),
366                local_filepath=local_filepath,
367                overwrite=overwrite,
368            )
369
370    def delete_folder(self, remote_folderpath: str) -> None:
371        """
372        Function:
373
374        - Deletes all files in a remote folder
375
376        Parameters:
377
378        - `remote_folderpath` (str): The path to the remote folder
379
380        Returns:
381
382        - `None`
383
384        Example:
385
386        ```python
387        from azure_utils.blob_storage import AZContainer
388        mycontainer = AZContainer(
389            account_url="my_account_url",
390            account_key="my_account_key",
391            container_name="my_container"
392        )
393        mycontainer.delete_folder(remote_folderpath='/path/to/folder/')
394        ```
395        """
396        validate_path(path=remote_folderpath, is_remote=True, is_folder=True)
397        storage_filenames = self.client.list_blob_names(
398            name_starts_with=remote_folderpath
399        )
400        for filename in storage_filenames:
401            blob = AZBlob(
402                blob_client=self.client.get_blob_client(blob=filename)
403            )
404            blob.delete()
405
406    def clear_local_meta(
407        self, local_folderpath: str, includsions: list = [".meta."]
408    ) -> None:
409        """
410        Function:
411
412        - Deletes all meta files in a local folder
413
414        Parameters:
415
416        - `local_folderpath` (str): The path to the local folder
417        - `includsions` (list): A list of strings to include in the deletion
418            - Optional: Defaults to `['.meta.']`
419
420        Returns:
421
422        - `None`
423
424        Example:
425
426        ```python
427        from azure_utils.blob_storage import AZContainer
428        mycontainer = AZContainer(
429            account_url="my_account_url",
430            account_key="my_account_key",
431            container_name="my_container"
432        )
433        mycontainer.clear_local_meta(local_folderpath='/path/to/folder/')
434        ```
435        """
436        validate_path(path=local_folderpath, is_remote=False, is_folder=True)
437        local_filepaths = [
438            str(i) for i in pathlib.Path(local_folderpath).glob("**/*")
439        ]
440        for filepath in [
441            i for i in local_filepaths if any([j in i for j in includsions])
442        ]:
443            os.remove(filepath)

AZContainer( account_url: str, account_key: str, container_name: str, show_progress: bool = False) View Source

 9    def __init__(
10        self,
11        account_url: str,
12        account_key: str,
13        container_name: str,
14        show_progress: bool = False,
15    ):
16        """
17        Function:
18
19        - Creates an AZContainer object
20
21        Parameters:
22
23        - `account_url` (str): The account url
24        - `account_key` (str): The account key
25        - `container_name` (str): The container name
26        - `show_progress` (bool): Whether show progress (log files) when uploading and downloading files
27
28        Returns:
29
30        - The AZContainer object
31
32        Example:
33
34        ```python
35        from azure_utils.blob_storage import AZContainer
36        mycontainer = AZContainer(
37            account_url="my_account_url",
38            account_key="my_account_key",
39            container_name="my_container"
40        )
41        print(mycontainer.list_files(remote_folderpath='/path/to/folder/')
42        # This will print a list of all files in the remote folder `/path/to/folder/`
43        ```
44        """
45        self.show_progress = show_progress
46        self.client = ContainerClient(
47            account_url=account_url,
48            container_name=container_name,
49            credential=account_key,
50        )

Function:

Creates an AZContainer object

Parameters:

account_url (str): The account url
account_key (str): The account key
container_name (str): The container name
show_progress (bool): Whether show progress (log files) when uploading and downloading files

Returns:

The AZContainer object

Example:

from azure_utils.blob_storage import AZContainer
mycontainer = AZContainer(
    account_url="my_account_url",
    account_key="my_account_key",
    container_name="my_container"
)
print(mycontainer.list_files(remote_folderpath='/path/to/folder/')
# This will print a list of all files in the remote folder `/path/to/folder/`

show_progress

client

def list_files(self, remote_folderpath: str) -> list: View Source

52    def list_files(self, remote_folderpath: str) -> list:
53        """
54        Function:
55
56        - Lists all files in a remote folder
57
58        Parameters:
59
60        - `remote_folderpath` (str): The path to the remote folder
61
62        Returns:
63
64        - A list of all files (as strings) in the remote folder (including subfolders)
65
66        Example:
67
68        ```python
69        from azure_utils.blob_storage import AZContainer
70        mycontainer = AZContainer(
71            account_url="my_account_url",
72            account_key="my_account_key",
73            container_name="my_container"
74        )
75        print(mycontainer.list_files(remote_folderpath='/path/to/folder/')
76        # This will print a list of all files in the remote folder `/path/to/folder/`
77        ```
78        """
79        validate_path(path=remote_folderpath, is_remote=True, is_folder=True)
80        return [
81            i.name
82            for i in self.client.list_blobs(name_starts_with=remote_folderpath)
83            if i.size > 0
84        ]

Function:

Lists all files in a remote folder

Parameters:

remote_folderpath (str): The path to the remote folder

Returns:

A list of all files (as strings) in the remote folder (including subfolders)

Example:

from azure_utils.blob_storage import AZContainer
mycontainer = AZContainer(
    account_url="my_account_url",
    account_key="my_account_key",
    container_name="my_container"
)
print(mycontainer.list_files(remote_folderpath='/path/to/folder/')
# This will print a list of all files in the remote folder `/path/to/folder/`

def upload_file( self, remote_filepath: str, local_filepath: str, overwrite: bool = False) -> None: View Source

 86    def upload_file(
 87        self, remote_filepath: str, local_filepath: str, overwrite: bool = False
 88    ) -> None:
 89        """
 90        Function:
 91
 92        - Uploads a file from local to remote
 93
 94        Parameters:
 95
 96        - `remote_filepath` (str): The path to the remote file
 97        - `local_filepath` (str): The path to the local file
 98        - `overwrite` (bool): Whether to overwrite the remote file if it already exists
 99            - Optional: Defaults to `False`
100
101        Returns:
102
103        - `None`
104
105        Example:
106
107        ```python
108        from azure_utils.blob_storage import AZContainer
109        mycontainer = AZContainer(
110            account_url="my_account_url",
111            account_key="my_account_key",
112            container_name="my_container"
113        )
114        mycontainer.upload_file(
115            remote_filepath='/path/to/file.txt',
116            local_filepath='/path/to/file.txt',
117            overwrite=True
118        )
119        # This will upload the local file `/path/to/file.txt` to the remote file `/path/to/file.txt`
120        ```
121        """
122        validate_path(path=local_filepath, is_remote=False, is_folder=False)
123        validate_path(path=remote_filepath, is_remote=True, is_folder=False)
124        if self.show_progress:
125            print(f"Uploading {local_filepath} to {remote_filepath}")
126        with open(local_filepath, "rb") as data:
127            blob = MetaBlob(
128                blob_client=self.client.upload_blob(
129                    name=remote_filepath, data=data, overwrite=overwrite
130                ),
131                filepath=local_filepath,
132            )
133        blob.update_meta()

Function:

Uploads a file from local to remote

Parameters:

remote_filepath (str): The path to the remote file
local_filepath (str): The path to the local file
overwrite (bool): Whether to overwrite the remote file if it already exists
- Optional: Defaults to False

Returns:

None

Example:

from azure_utils.blob_storage import AZContainer
mycontainer = AZContainer(
    account_url="my_account_url",
    account_key="my_account_key",
    container_name="my_container"
)
mycontainer.upload_file(
    remote_filepath='/path/to/file.txt',
    local_filepath='/path/to/file.txt',
    overwrite=True
)
# This will upload the local file `/path/to/file.txt` to the remote file `/path/to/file.txt`

def download_file( self, remote_filepath: str, local_filepath: str, overwrite: bool = False, smart_sync: bool = False) -> None: View Source

135    def download_file(
136        self,
137        remote_filepath: str,
138        local_filepath: str,
139        overwrite: bool = False,
140        smart_sync: bool = False,
141    ) -> None:
142        """
143        Function:
144
145        - Downloads a file from remote to local
146
147        Parameters:
148
149        - `remote_filepath` (str): The path to the remote file
150        - `local_filepath` (str): The path to the local file
151        - `overwrite` (bool): Whether to overwrite the local file if it already exists
152            - Optional: Defaults to `False`
153        - `smart_sync` (bool): Whether to skip downloading if the remote etag and md5 hash match the local meta file
154            - Optional: Defaults to `False`
155
156        Returns:
157
158        - `None`
159
160        Example:
161
162        ```python
163        from azure_utils.blob_storage import AZContainer
164        mycontainer = AZContainer(
165            account_url="my_account_url",
166            account_key="my_account_key",
167            container_name="my_container"
168        )
169        mycontainer.download_file(
170            remote_filepath='/path/to/file.txt',
171            local_filepath='/path/to/file.txt',
172            overwrite=True,
173            smart_sync=True
174        )
175        # This will download the remote file `/path/to/file.txt` to the local file `/path/to/file.txt`
176        ```
177        """
178        validate_path(path=local_filepath, is_remote=False, is_folder=False)
179        validate_path(path=remote_filepath, is_remote=True, is_folder=False)
180        if self.show_progress:
181            print(f"Downloading {remote_filepath} to {local_filepath}")
182        blob = MetaBlob(
183            blob_client=self.client.get_blob_client(blob=remote_filepath),
184            filepath=local_filepath,
185            smart_sync=smart_sync,
186            overwrite=overwrite,
187        )
188        blob.download()

Function:

Downloads a file from remote to local

Parameters:

remote_filepath (str): The path to the remote file
local_filepath (str): The path to the local file
overwrite (bool): Whether to overwrite the local file if it already exists
- Optional: Defaults to False
smart_sync (bool): Whether to skip downloading if the remote etag and md5 hash match the local meta file
- Optional: Defaults to False

Returns:

None

Example:

from azure_utils.blob_storage import AZContainer
mycontainer = AZContainer(
    account_url="my_account_url",
    account_key="my_account_key",
    container_name="my_container"
)
mycontainer.download_file(
    remote_filepath='/path/to/file.txt',
    local_filepath='/path/to/file.txt',
    overwrite=True,
    smart_sync=True
)
# This will download the remote file `/path/to/file.txt` to the local file `/path/to/file.txt`

def delete_file(self, remote_filepath: str) -> None: View Source

190    def delete_file(self, remote_filepath: str) -> None:
191        """
192        Function:
193
194        - Deletes a file from remote
195
196        Parameters:
197
198        - `remote_filepath` (str): The path to the remote file
199
200        Returns:
201
202        - `None`
203
204        Example:
205        ```python
206        from azure_utils.blob_storage import AZContainer
207        mycontainer = AZContainer(
208            account_url="my_account_url",
209            account_key="my_account_key",
210            container_name="my_container"
211        )
212        mycontainer.delete_file(remote_filepath='/path/to/file.txt')
213        ```
214        """
215        validate_path(path=remote_filepath, is_remote=True, is_folder=False)
216        if self.show_progress:
217            print(f"Deleting {remote_filepath}")
218        blob = AZBlob(
219            blob_client=self.client.get_blob_client(blob=remote_filepath)
220        )
221        blob.delete()

Function:

Deletes a file from remote

Parameters:

remote_filepath (str): The path to the remote file

Returns:

None

Example:

from azure_utils.blob_storage import AZContainer
mycontainer = AZContainer(
    account_url="my_account_url",
    account_key="my_account_key",
    container_name="my_container"
)
mycontainer.delete_file(remote_filepath='/path/to/file.txt')

def sync_to_local( self, remote_folderpath: str, local_folderpath: str, overwrite: bool = False, smart_sync: bool = False, removal: bool = False) -> None: View Source

223    def sync_to_local(
224        self,
225        remote_folderpath: str,
226        local_folderpath: str,
227        overwrite: bool = False,
228        smart_sync: bool = False,
229        removal: bool = False,
230    ) -> None:
231        """
232        Function:
233
234        - Sync all files from a remote folder to a local folder
235
236        Parameters:
237
238        - `remote_folderpath` (str): The path to the remote folder
239        - `local_folderpath` (str): The path to the local folder
240        - `overwrite` (bool): Whether to overwrite the local file if it already exists
241            - Optional: Defaults to `False`
242        - `smart_sync` (bool): Whether to skip downloading if the remote etag and md5 hash match the local meta file
243            - Optional: Defaults to `False`
244        - `removal` (bool): Whether to remove local files that do not exist in the remote folder
245            - Optional: Defaults to `False`
246
247        Returns:
248
249        - `None`
250
251        Example:
252
253        ```python
254        from azure_utils.blob_storage import AZContainer
255        mycontainer = AZContainer(
256            account_url="my_account_url",
257            account_key="my_account_key",
258            container_name="my_container"
259        )
260        mycontainer.sync_to_local(
261            remote_folderpath='/path/to/folder/',
262            local_folderpath='/path/to/folder/',
263            overwrite=True,
264            smart_sync=True,
265            removal=True
266        )
267        # This will sync the remote folder `/path/to/folder/` to the local folder `/path/to/folder/`
268        # It will also remove any local files that do not exist in the remote folder
269        ```
270        """
271        validate_path(path=local_folderpath, is_remote=False, is_folder=True)
272        validate_path(path=remote_folderpath, is_remote=True, is_folder=True)
273        blobs = [
274            i
275            for i in self.client.list_blobs(name_starts_with=remote_folderpath)
276            if i.size > 0
277        ]
278        for blobProperty in blobs:
279            blob = MetaBlob(
280                blob_client=None,
281                filepath=local_folderpath
282                + blobProperty.name.replace(remote_folderpath, "/", 1),
283                smart_sync=smart_sync,
284                remote_etag=blobProperty.etag,
285                overwrite=overwrite,
286            )
287            if blob.block_download:
288                continue
289            blob.blob_client = self.client.get_blob_client(
290                blob=blobProperty.name
291            )
292            blob.download()
293        if removal:
294            posix_folderpath = str(pathlib.PurePosixPath(local_folderpath))
295            if posix_folderpath == "/":
296                raise Exception("Cannot do removal in the root folder")
297            storage_filepaths = set(
298                [i.name.replace(remote_folderpath, "") for i in blobs]
299            )
300            for local_filepath in [
301                str(i) for i in pathlib.Path(local_folderpath).glob("**/*")
302            ]:
303                posix_filepath = str(pathlib.PurePosixPath(local_filepath))
304                if (
305                    posix_filepath.replace(posix_folderpath, "").replace(
306                        ".meta.", ""
307                    )
308                    not in storage_filepaths
309                ):
310                    os.remove(local_filepath)

Function:

Sync all files from a remote folder to a local folder

Parameters:

remote_folderpath (str): The path to the remote folder
local_folderpath (str): The path to the local folder
overwrite (bool): Whether to overwrite the local file if it already exists
- Optional: Defaults to False
smart_sync (bool): Whether to skip downloading if the remote etag and md5 hash match the local meta file
- Optional: Defaults to False
removal (bool): Whether to remove local files that do not exist in the remote folder
- Optional: Defaults to False

Returns:

None

Example:

from azure_utils.blob_storage import AZContainer
mycontainer = AZContainer(
    account_url="my_account_url",
    account_key="my_account_key",
    container_name="my_container"
)
mycontainer.sync_to_local(
    remote_folderpath='/path/to/folder/',
    local_folderpath='/path/to/folder/',
    overwrite=True,
    smart_sync=True,
    removal=True
)
# This will sync the remote folder `/path/to/folder/` to the local folder `/path/to/folder/`
# It will also remove any local files that do not exist in the remote folder

def sync_to_remote( self, remote_folderpath: str, local_folderpath: str, overwrite: bool = False, omissions: list = ['.meta.']) -> None: View Source

312    def sync_to_remote(
313        self,
314        remote_folderpath: str,
315        local_folderpath: str,
316        overwrite: bool = False,
317        omissions: list = [".meta."],
318    ) -> None:
319        """
320        Function:
321
322        - Sync all files from a local folder to a remote folder
323
324        Parameters:
325
326        - `remote_folderpath` (str): The path to the remote folder
327        - `local_folderpath` (str): The path to the local folder
328        - `overwrite` (bool): Whether to overwrite the remote file if it already exists
329            - Optional: Defaults to `False`
330        - `omissions` (list): A list of strings to omit from the upload
331            - Optional: Defaults to `['.meta.']`
332
333        Returns:
334
335        - `None`
336
337        Example:
338        ```python
339        from azure_utils.blob_storage import AZContainer
340        mycontainer = AZContainer(
341            account_url="my_account_url",
342            account_key="my_account_key",
343            container_name="my_container"
344        )
345        mycontainer.sync_to_remote(
346            remote_folderpath='/path/to/folder/',
347            local_folderpath='/path/to/folder/',
348        )
349        # This will sync the local folder `/path/to/folder/` to the remote folder `/path/to/folder/`
350        ```
351        """
352        validate_path(path=local_folderpath, is_remote=False, is_folder=True)
353        validate_path(path=remote_folderpath, is_remote=True, is_folder=True)
354        local_filepaths = [
355            str(i)
356            for i in pathlib.Path(local_folderpath).glob("**/*")
357            if i.is_file()
358        ]
359        local_filepaths = [
360            i for i in local_filepaths if not any([j in i for j in omissions])
361        ]
362        for local_filepath in local_filepaths:
363            self.upload_file(
364                remote_filepath=remote_folderpath
365                + local_filepath.replace(local_folderpath, ""),
366                local_filepath=local_filepath,
367                overwrite=overwrite,
368            )

Function:

Sync all files from a local folder to a remote folder

Parameters:

remote_folderpath (str): The path to the remote folder
local_folderpath (str): The path to the local folder
overwrite (bool): Whether to overwrite the remote file if it already exists
- Optional: Defaults to False
omissions (list): A list of strings to omit from the upload
- Optional: Defaults to ['.meta.']

Returns:

None

Example:

from azure_utils.blob_storage import AZContainer
mycontainer = AZContainer(
    account_url="my_account_url",
    account_key="my_account_key",
    container_name="my_container"
)
mycontainer.sync_to_remote(
    remote_folderpath='/path/to/folder/',
    local_folderpath='/path/to/folder/',
)
# This will sync the local folder `/path/to/folder/` to the remote folder `/path/to/folder/`

def delete_folder(self, remote_folderpath: str) -> None: View Source

370    def delete_folder(self, remote_folderpath: str) -> None:
371        """
372        Function:
373
374        - Deletes all files in a remote folder
375
376        Parameters:
377
378        - `remote_folderpath` (str): The path to the remote folder
379
380        Returns:
381
382        - `None`
383
384        Example:
385
386        ```python
387        from azure_utils.blob_storage import AZContainer
388        mycontainer = AZContainer(
389            account_url="my_account_url",
390            account_key="my_account_key",
391            container_name="my_container"
392        )
393        mycontainer.delete_folder(remote_folderpath='/path/to/folder/')
394        ```
395        """
396        validate_path(path=remote_folderpath, is_remote=True, is_folder=True)
397        storage_filenames = self.client.list_blob_names(
398            name_starts_with=remote_folderpath
399        )
400        for filename in storage_filenames:
401            blob = AZBlob(
402                blob_client=self.client.get_blob_client(blob=filename)
403            )
404            blob.delete()

Function:

Deletes all files in a remote folder

Parameters:

remote_folderpath (str): The path to the remote folder

Returns:

None

Example:

from azure_utils.blob_storage import AZContainer
mycontainer = AZContainer(
    account_url="my_account_url",
    account_key="my_account_key",
    container_name="my_container"
)
mycontainer.delete_folder(remote_folderpath='/path/to/folder/')

def clear_local_meta(self, local_folderpath: str, includsions: list = ['.meta.']) -> None: View Source

406    def clear_local_meta(
407        self, local_folderpath: str, includsions: list = [".meta."]
408    ) -> None:
409        """
410        Function:
411
412        - Deletes all meta files in a local folder
413
414        Parameters:
415
416        - `local_folderpath` (str): The path to the local folder
417        - `includsions` (list): A list of strings to include in the deletion
418            - Optional: Defaults to `['.meta.']`
419
420        Returns:
421
422        - `None`
423
424        Example:
425
426        ```python
427        from azure_utils.blob_storage import AZContainer
428        mycontainer = AZContainer(
429            account_url="my_account_url",
430            account_key="my_account_key",
431            container_name="my_container"
432        )
433        mycontainer.clear_local_meta(local_folderpath='/path/to/folder/')
434        ```
435        """
436        validate_path(path=local_folderpath, is_remote=False, is_folder=True)
437        local_filepaths = [
438            str(i) for i in pathlib.Path(local_folderpath).glob("**/*")
439        ]
440        for filepath in [
441            i for i in local_filepaths if any([j in i for j in includsions])
442        ]:
443            os.remove(filepath)

Function:

Deletes all meta files in a local folder

Parameters:

local_folderpath (str): The path to the local folder
includsions (list): A list of strings to include in the deletion
- Optional: Defaults to ['.meta.']

Returns:

None

Example:

from azure_utils.blob_storage import AZContainer
mycontainer = AZContainer(
    account_url="my_account_url",
    account_key="my_account_key",
    container_name="my_container"
)
mycontainer.clear_local_meta(local_folderpath='/path/to/folder/')