Fix cloud_sync

setup.py

@@ -37,6 +37,7 @@
         "ugbio_utils/src/cnv/ugbio_cnv/plot_cnv_results.py",
         "ugbio_utils/src/cnv/ugbio_cnv/annotate_FREEC_segments.py",
         "ugvc/pipelines/correct_genotypes_by_imputation.py",
+        "ugvc/utils/cloud_sync.py",
     ],
     package_data={
         "ugvc": [

setup/environment.yml

@@ -1,8 +1,7 @@
 name: genomics.py3
 channels:
-  - conda-forge
   - bioconda
-  - r
+  - conda-forge
   - defaults
 dependencies:
   - _libgcc_mutex=0.1=conda_forge
@@ -62,7 +61,7 @@ dependencies:
   - cached_property=1.5.2=pyha770c72_1
   - cachetools=5.3.3=pyhd8ed1ab_0
   - cairo=1.18.0=h3faef2a_0
-  - certifi=2024.7.4=pyhd8ed1ab_0
+  - certifi=2024.8.30=py310h06a4308_0
   - cffi=1.16.0=py310h2fee648_0
   - cfgv=3.3.1=pyhd8ed1ab_0
   - charset-normalizer=3.3.2=pyhd8ed1ab_0
@@ -86,6 +85,7 @@ dependencies:
   - distributed=2024.2.1=pyhd8ed1ab_0
   - dnspython=2.6.1=pyhd8ed1ab_1
   - entrypoints=0.4=pyhd8ed1ab_0
+  - et_xmlfile=1.1.0=py310h06a4308_0
   - exceptiongroup=1.2.0=pyhd8ed1ab_2
   - executing=2.0.1=pyhd8ed1ab_0
   - expat=2.6.2=h59595ed_0
@@ -294,6 +294,7 @@ dependencies:
   - numpy=1.26.4=py310hb13e2d6_0
   - openjdk=17.0.11=h24d6bf4_0
   - openjpeg=2.5.2=h488ebb8_0
+  - openpyxl=3.1.5=py310h5eee18b_0
   - openssl=3.3.1=h4bc722e_2
   - orc=2.0.1=h17fec99_1
   - overrides=7.7.0=pyhd8ed1ab_0
@@ -479,4 +480,5 @@ dependencies:
       - sigprofilermatrixgenerator==1.2.26
       - sigprofilerplotting==1.3.23
       - simppl==1.0.7
-prefix: /home/ilyas/anaconda3/envs/genomics.py3
+      - ugvc==0.24
+prefix: /home/ubuntu/miniconda3/envs/genomics.py3

ugvc/utils/cloud_sync.py

@@ -1,3 +1,5 @@
+#!/env/python
+
 from __future__ import annotations
 
 import argparse
@@ -48,7 +50,12 @@ def optional_cloud_sync(
 ):
     if cloud_path_in.startswith("gs://") or cloud_path_in.startswith("s3://"):
         return cloud_sync(
-            cloud_path_in, local_dir_in, print_output, force_download, raise_error_is_file_exists, dry_run
+            cloud_path_in,
+            local_dir_in,
+            print_output,
+            force_download,
+            raise_error_is_file_exists,
+            dry_run,
         )
     return cloud_path_in
 
@@ -60,6 +67,7 @@ def cloud_sync(
     force_download=False,
     raise_error_is_file_exists=False,
     dry_run=False,
+    verbose=False,
 ):
     """Download a file from the cloud to a respective local directory with the same name
 
@@ -77,6 +85,9 @@ def cloud_sync(
         If True and local file already exists, raise ValueError (default False)
     dry_run: bool, optional
         If True, return local path without downloading (default False)
+    verbose: bool, optional
+        If True, print more log messages (default False)
+
     :raises NotADirectoryError: Not a directory
     :raises FileExistsError: Target local file exists
     :raises NotImplementedError: Unsupported cloud service
@@ -105,13 +116,15 @@ def cloud_sync(
         return local_path
     if not force_download and os.path.isfile(local_path):
         if raise_error_is_file_exists:
-            raise FileExistsError(f"target local file {local_path} exists")
+            raise FileExistsError(f"target local file {local_path} exists\n")
         if print_output:
-            sys.stdout.write(f"Local file {local_path} already exists, skipping...\n")
+            sys.stdout.write(
+                f"Local file {local_path} already exists, skipping...\n" if verbose else local_path + os.linesep
+            )
     else:
         try:
             if print_output:
-                sys.stdout.write(f"Downloading to {local_path}\n")
+                sys.stdout.write(("Downloading to " if verbose else "") + local_path + os.linesep)
             os.makedirs(os.path.dirname(local_path), exist_ok=True)
             if cloud_service == "gs":
                 download_from_gs(bucket, blob, local_path)
@@ -121,7 +134,8 @@ def cloud_sync(
                 raise NotImplementedError()
         except KeyboardInterrupt:
             if os.path.isfile(local_path):
-                sys.stdout.write(f"Keyboard Interrupt - removing incomplete file {local_path}\n")
+                if verbose:
+                    sys.stdout.write(f"Keyboard Interrupt - removing incomplete file {local_path}\n")
                 os.remove(local_path)
             raise
 
@@ -137,6 +151,13 @@ def parse_args(argv):
         default="/data",
         help="local directory the files will sync to",
     )
+    parser.add_argument(
+        "--no-output",
+        action="store_false",
+        dest="print_output",
+        help="do not print output",
+    )
+    parser.add_argument("--verbose", action="store_true", help="print more log messages")
     args = parser.parse_args(argv[1:])
     return args
 
@@ -146,6 +167,12 @@ def run(argv: list[str]):
     args = parse_args(argv)
     cloud_path = args.cloud_path
     local_dir = args.local_dir
+    print_output = args.print_output
+    verbose = args.verbose and print_output
     if not os.path.isdir(local_dir):
         raise ValueError(f"local_dir {local_dir} does not exist")
-    cloud_sync(cloud_path, local_dir, print_output=True)
+    cloud_sync(cloud_path, local_dir, print_output=print_output, verbose=verbose)
+
+
+if __name__ == "__main__":
+    run(sys.argv)