diff --git a/.gitignore b/.gitignore index 71037206..25fea9d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ +*.sublime-workspace *.code-workspace +.vscode +__pycache__/ *.pyc -*.sublime-workspace +*~ *.swp .conda .idea -.vscode -__pycache__ diff --git a/README.md b/README.md index ca28858c..8ffcdb34 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,11 @@ -# SimSys Scripts +# Simulation Systems Scripts -This repository contains support scripts that are common across the many simulation and modelling codes owned by the Met Office. -Particularily those owned and maintained by the SSD team. +[![Checks](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/lint.yml/badge.svg)](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/lint.yml) +[![CodeQL](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/github-code-scanning/codeql) -Also contains a copy of script_updater.sh which is intended to live in the fcm repositories to pull from -this repo. +This repository contains support scripts that are common across the many +simulation and modelling codes owned by the Met Office. Particularly those +owned and maintained by the Simulation Systems and Deployment (SSD) team. + +Also contains a copy of `script_updater.sh` which is intended to live in the +fcm repositories to pull from this repository. diff --git a/fcm_bdiff.py b/fcm_bdiff.py index 6a9761e7..7e20a8e4 100644 --- a/fcm_bdiff.py +++ b/fcm_bdiff.py @@ -9,8 +9,9 @@ run tests on based on the branch-difference (to allow checking of only files which a developer has actually modified on their branch) """ -import re + import os +import re import subprocess import time @@ -20,10 +21,11 @@ class FCMError(Exception): """ Exception class for FCM commands """ + def __str__(self): - return ("\nFCM command: \"{0:s}\"" - "\nFailed with error: \"{1:s}\"" - .format(" ".join(self.args[0]), self.args[1].strip())) + return '\nFCM command: "{0:s}"\nFailed with error: "{1:s}"'.format( + " ".join(self.args[0]), self.args[1].strip() + ) # ------------------------------------------------------------------------------ @@ -32,16 +34,20 @@ def is_trunk(url): Given an FCM url, returns True if it appears to be pointing to the UM main trunk """ - search = re.search(r""" + search = re.search( + r""" (svn://fcm\d+/\w+_svn/\w+/trunk| .*/svn/[\w\.]+/\w+/trunk| ..*_svn/\w+/trunk) - """, url, flags=re.VERBOSE) + """, + url, + flags=re.VERBOSE, + ) return search is not None # ------------------------------------------------------------------------------ -def text_decoder(bytes_type_string, codecs=['utf8', 'cp1252']): +def text_decoder(bytes_type_string, codecs=["utf8", "cp1252"]): """ Given a bytes type string variable, attempt to decode it using the codecs listed. @@ -127,18 +133,20 @@ def get_branch_diff_filenames(branch=".", path_override=None): # Strip whitespace, and remove blank lines while turning the output into # a list of strings. bdiff_files = [x.strip() for x in bdiff.split("\n") if x.strip()] - bdiff_files = [bfile.split()[1] for bfile in bdiff_files - if bfile.split()[0].strip() == "M" or - bfile.split()[0].strip() == "A"] + bdiff_files = [ + bfile.split()[1] + for bfile in bdiff_files + if bfile.split()[0].strip() == "M" or bfile.split()[0].strip() == "A" + ] # Convert the file paths to be relative to the current URL; to do this # construct the base path of the trunk URL and compare it to the results # of the bdiff command above repos_root = get_repository_root(info) - relative_paths = [os.path.relpath(bfile, - os.path.join(repos_root, - "main", "trunk")) - for bfile in bdiff_files] + relative_paths = [ + os.path.relpath(bfile, os.path.join(repos_root, "main", "trunk")) + for bfile in bdiff_files + ] # These relative paths can be joined to an appropriate base to complete # the filenames to return @@ -169,9 +177,9 @@ def run_fcm_command(command, max_retries, snooze): """ retries = 0 while True: - output = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + output = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) _ = output.wait() if output.returncode == 0: return text_decoder(output.stdout.read()) @@ -198,8 +206,7 @@ def use_mirror(branch): if mirror_key in os.environ: branch = os.environ[mirror_key] retries = 2 - print("[INFO] Switching branch used for fcm command to : {0:}".format( - branch)) + print(f"[INFO] Switching branch used for fcm command to: {branch}") else: retries = 0 return branch, retries @@ -211,8 +218,9 @@ def get_repository_root(branch_info): Given the raw output from an fcm binfo command - which can be retrieved by calling get_branch_info() - returns the Repository Root field """ - repos_root = re.search(r"^Repository Root:\s*(?P.*)\s*$", - branch_info, flags=re.MULTILINE) + repos_root = re.search( + r"^Repository Root:\s*(?P.*)\s*$", branch_info, flags=re.MULTILINE + ) if repos_root: repos_root = repos_root.group("url") else: @@ -226,8 +234,9 @@ def get_branch_parent(branch_info): Given the raw output from an fcm binfo command - which can be retrieved by calling get_branch_info() - returns the Branch Parent Field """ - parent = re.search(r"^Branch Parent:\s*(?P.*)$", branch_info, - flags=re.MULTILINE) + parent = re.search( + r"^Branch Parent:\s*(?P.*)$", branch_info, flags=re.MULTILINE + ) if parent: parent = parent.group("parent") else: diff --git a/kgo_updates/kgo_update/kgo_update.py b/kgo_updates/kgo_update/kgo_update.py index 4224bf5b..1b9435f0 100755 --- a/kgo_updates/kgo_update/kgo_update.py +++ b/kgo_updates/kgo_update/kgo_update.py @@ -31,15 +31,14 @@ """ +import argparse import os import re -import sys import sqlite3 -import argparse import subprocess +import sys from collections import defaultdict - # Global criteria for updating - any statuses matched by this list will # be treated as "failed" for the purposes of updating. When running a # "new version" update this list will be updated to include the "OK" @@ -48,8 +47,8 @@ def banner(message): - '''Print a simple banner message''' - return "{0}\n* {1} *\n{0}".format("%"*(len(message)+4), message) + """Print a simple banner message""" + return "{0}\n* {1} *\n{0}".format("%" * (len(message) + 4), message) def confirm(message, skip=False): @@ -91,9 +90,9 @@ def write_update_script(kgo_dirs, new_dirname, script): # Write a sub-header for this KGO directory to help visually split # up the file (for when the user checks it by eye) - script.write("#"*(len(new_kgo_dir)+4)+"\n") + script.write("#" * (len(new_kgo_dir) + 4) + "\n") script.write("# {0} #\n".format(new_kgo_dir)) - script.write("#"*(len(new_kgo_dir)+4)+"\n") + script.write("#" * (len(new_kgo_dir) + 4) + "\n") script.write(f"echo 'Installing {new_kgo_dir}'\n\n") # Build up several different text sections in the upcoming loop @@ -125,8 +124,7 @@ def write_update_script(kgo_dirs, new_dirname, script): # without a source here must have been retired from # the tests continue - keep_description.append( - "# * {0}".format(kgo_file)) + keep_description.append("# * {0}".format(kgo_file)) # Construct the paths old_file = os.path.join(kgo_dir, kgo_file) @@ -134,23 +132,22 @@ def write_update_script(kgo_dirs, new_dirname, script): keep_commands.append( "echo 'ln -s {0} {1}'".format( - os.path.relpath(old_file, - os.path.dirname(new_file)), - new_file)) + os.path.relpath(old_file, os.path.dirname(new_file)), + new_file, + ) + ) keep_commands.append( "ln -s {0} {1}".format( - os.path.relpath(old_file, - os.path.dirname(new_file)), - new_file)) + os.path.relpath(old_file, os.path.dirname(new_file)), + new_file, + ) + ) else: # Files from the suite should be copied from the suite - copy_description.append( - "# * {0}".format(kgo_file)) + copy_description.append("# * {0}".format(kgo_file)) new_file = os.path.join(new_kgo_dir, kgo_file) - copy_commands.append( - "echo 'cp {0} {1}'".format(source, new_file)) - copy_commands.append( - "cp {0} {1}".format(source, new_file)) + copy_commands.append("echo 'cp {0} {1}'".format(source, new_file)) + copy_commands.append("cp {0} {1}".format(source, new_file)) # In this case more disk-space is needed, so update # the running total for reporting later total_filesize += os.path.getsize(source) @@ -181,10 +178,14 @@ def report_space_required(total_filesize, skip=False): units.pop() unit = units[-1] - print(banner("Update will require: {0:.2f} {1} of disk space" - .format(total_filesize, unit))) - if not confirm("Please confirm this much space is available (y/n)? ", - skip=skip): + print( + banner( + "Update will require: {0:.2f} {1} of disk space".format( + total_filesize, unit + ) + ) + ) + if not confirm("Please confirm this much space is available (y/n)? ", skip=skip): sys.exit("Aborting...") @@ -193,8 +194,7 @@ def add_untested_kgo_files(kgo_dirs): for kgo_dir, update_dict in kgo_dirs.items(): for path, _, filenames in os.walk(kgo_dir): for filename in filenames: - kgo_file = ( - os.path.relpath(os.path.join(path, filename), kgo_dir)) + kgo_file = os.path.relpath(os.path.join(path, filename), kgo_dir) if kgo_file not in update_dict: kgo_dirs[kgo_dir][kgo_file] = None return kgo_dirs @@ -217,9 +217,11 @@ def group_comparisons_by_dir(comparisons, skip=False): # If it looks like this KGO file never existed at all, let the user # decide what they want to happen if not os.path.exists(kgo_file) and status.strip() in UPDATE_CRITERIA: - if not confirm("KGO file {0} doesn't appear to exist, should we " - "install it from the suite (y/n)? " - .format(kgo_file), skip=skip): + if not confirm( + "KGO file {0} doesn't appear to exist, should we " + "install it from the suite (y/n)? ".format(kgo_file), + skip=skip, + ): # Note this is negated - i.e. if the user doesn't want to # include this new file, we skip it and move on continue @@ -227,15 +229,18 @@ def group_comparisons_by_dir(comparisons, skip=False): # Extract the base KGO directory by moving backwards through the path # until a directory that matches the KGO directory naming style basedir = os.path.dirname(kgo_file) - while (basedir != "/" and not - re.match(r".*((vn|\d+\.)\d+\.\d+(_t\d+|))$", basedir)): + while basedir != "/" and not re.match( + r".*((vn|\d+\.)\d+\.\d+(_t\d+|))$", basedir + ): basedir = os.path.dirname(basedir) # If the above goes wrong it will eventually hit root; if this happens # we cannot continue as something is seriously not right if basedir == "/": - msg = ("Problem locating KGO directory - " - "is this actually a KGO file?\n {0}") + msg = ( + "Problem locating KGO directory - " + "is this actually a KGO file?\n {0}" + ) sys.exit(msg.format(kgo_file)) # Otherwise add the result to the list - for each entry we store the @@ -247,8 +252,10 @@ def group_comparisons_by_dir(comparisons, skip=False): # and if it has perhaps we can do some additional checking # (for instance, are the changes in answers the same?) relative_kgo_path = os.path.relpath(kgo_file, basedir) - if (relative_kgo_path in kgo_dirs[basedir] and - kgo_dirs[basedir][relative_kgo_path] != suite_file): + if ( + relative_kgo_path in kgo_dirs[basedir] + and kgo_dirs[basedir][relative_kgo_path] != suite_file + ): # Or not... it isn't clear what could be checked here continue @@ -259,8 +266,9 @@ def group_comparisons_by_dir(comparisons, skip=False): def get_all_kgo_comparisons(conn): "Retrieve all comparisons related to KGO tasks" - res = conn.execute("SELECT comp_task, kgo_file, suite_file, " - "status, comparison FROM comparisons") + res = conn.execute( + "SELECT comp_task, kgo_file, suite_file, status, comparison FROM comparisons" + ) return res.fetchall() @@ -269,12 +277,16 @@ def check_for_incomplete_tasks(conn, skip=False): res = conn.execute("SELECT task_name FROM tasks WHERE completed==?", (1,)) failed_tasks = res.fetchall() if len(failed_tasks) > 0: - print("WARNING - Some comparisons are either still running or " - "failed to complete properly:") + print( + "WARNING - Some comparisons are either still running or " + "failed to complete properly:" + ) for task in failed_tasks: - print(" * "+task[0]) - print("Please investigate this manually - no KGO updates will be " - "made for any of the tasks listed above") + print(" * " + task[0]) + print( + "Please investigate this manually - no KGO updates will be " + "made for any of the tasks listed above" + ) if not confirm("Continue anyway (y/n)? ", skip=skip): sys.exit() @@ -304,17 +316,14 @@ def get_site(): Function to return the site this is being run at using rose config """ command = ["rose", "config", "rose-stem", "automatic-options"] - cmd = subprocess.Popen( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cmd = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = cmd.communicate() retcode = cmd.returncode if retcode == 0: - site = stdout.decode('utf-8').split("=")[1].strip() + site = stdout.decode("utf-8").split("=")[1].strip() print("Found site via rose-config: {0}\n".format(site)) else: - sys.exit( - "No site was detected via rose config - this setting is required" - ) + sys.exit("No site was detected via rose config - this setting is required") return site @@ -332,16 +341,14 @@ def get_variables_file_path(suite_dir, site, platform, variables_extension): if platform is not None: vars_file = f"variables_{platform}{variables_extension}" variables_path = os.path.join(site_path, vars_file) - print("INFO: Looking for a kgo variables file at " - + variables_path) + print("INFO: Looking for a kgo variables file at " + variables_path) if os.path.exists(variables_path): return variables_path # Search for a variables. file if a platform one doesn't exist, eg. VM site vars_file = f"variables{variables_extension}" variables_path = os.path.join(site_path, vars_file) - print("INFO: Looking for a kgo variables file at " - + variables_path) + print("INFO: Looking for a kgo variables file at " + variables_path) if os.path.exists(variables_path): return variables_path @@ -349,42 +356,40 @@ def get_variables_file_path(suite_dir, site, platform, variables_extension): def update_variables_rc( - suite_dir, - kgo_dirs, - new_kgo_dir, - site, - platform, - variables_extension, - skip=False - ): + suite_dir, + kgo_dirs, + new_kgo_dir, + site, + platform, + variables_extension, + skip=False, +): """ Create an updated copy of the variables.rc with the KGO variables for any changed jobs updated """ # Attempt to get a copy of the variables.rc - variables_rc = get_variables_file_path(suite_dir, - site, - platform, - variables_extension) + variables_rc = get_variables_file_path( + suite_dir, site, platform, variables_extension + ) # Create a file to hold the new variables.rc variables_rc_new = os.path.expanduser( f"~/variables{variables_extension}_{new_kgo_dir}" ) if os.path.exists(variables_rc_new): - print("WARNING: New variables.rc file for this update already exists " - "at {0} and will be overwritten".format(variables_rc_new)) - if not confirm("Okay to overwrite variables.rc file (y/n)? ", - skip=skip): + print( + "WARNING: New variables.rc file for this update already exists " + "at {0} and will be overwritten".format(variables_rc_new) + ) + if not confirm("Okay to overwrite variables.rc file (y/n)? ", skip=skip): sys.exit("Aborting...") # Get the KGO variable names that need updating kgo_vars = [] for kgo_dir in kgo_dirs.keys(): - kgo_vars.append( - os.path.basename( - os.path.dirname(kgo_dir)).upper()) + kgo_vars.append(os.path.basename(os.path.dirname(kgo_dir)).upper()) # Get the suffix of the new directory if "_" not in new_kgo_dir: @@ -399,10 +404,13 @@ def update_variables_rc( # named in the database pattern = re.search(r'\s*"(\S+)"\s*:\s*BASE', line) if pattern and pattern.group(1).upper() in kgo_vars: - vrc_new.write(re.sub( - r':\s*BASE.*', - r': BASE~"_{}",'.format(ticket_suffix), - line)) + vrc_new.write( + re.sub( + r":\s*BASE.*", + r': BASE~"_{}",'.format(ticket_suffix), + line, + ) + ) else: # Otherwise just write the line unaltered vrc_new.write(line) @@ -417,9 +425,9 @@ def main(): # adds spaces between the option help text class BlankLinesHelpFormatter(argparse.HelpFormatter): "Formatter which adds blank lines between options" + def _split_lines(self, text, width): - return super( - BlankLinesHelpFormatter, self)._split_lines(text, width) + [''] + return super(BlankLinesHelpFormatter, self)._split_lines(text, width) + [""] parser = argparse.ArgumentParser( usage="%(prog)s [--new-release]", @@ -432,55 +440,69 @@ def _split_lines(self, text, width): is used) """, formatter_class=BlankLinesHelpFormatter, - ) + ) - parser.add_argument('--new-release', - help="if set, indicates that the task being " - "performed is installing a full set of new KGO " - "for a major model release instead of simply " - "updating failed KGO tasks.", - action="store_true", - ) + parser.add_argument( + "--new-release", + help="if set, indicates that the task being " + "performed is installing a full set of new KGO " + "for a major model release instead of simply " + "updating failed KGO tasks.", + action="store_true", + ) - parser.add_argument('--non-interactive', - help="if set, no questions will be asked and " - "the script will proceed without any checking " - "or warnings. Use with caution.", - action="store_true", - ) + parser.add_argument( + "--non-interactive", + help="if set, no questions will be asked and " + "the script will proceed without any checking " + "or warnings. Use with caution.", + action="store_true", + ) - parser.add_argument('-S', "--suite-name", - help="specifies the (cylc-run dir) name of " - "the suite containing the KGO database; for " - "use with non-interactive mode. If not given " - "non-interactive mode will try to take " - "this from the environment ($CYLC_SUITE_NAME)." - ) + parser.add_argument( + "-S", + "--suite-name", + help="specifies the (cylc-run dir) name of " + "the suite containing the KGO database; for " + "use with non-interactive mode. If not given " + "non-interactive mode will try to take " + "this from the environment ($CYLC_SUITE_NAME).", + ) - parser.add_argument('-U', "--suite-user", - help="specifies the (cylc-run dir) username " - "where the KGO database suite can be found; for " - "use with non-interactive mode. If not given " - "non-interactive mode will try to take " - "this from the environment ($USER)." - ) + parser.add_argument( + "-U", + "--suite-user", + help="specifies the (cylc-run dir) username " + "where the KGO database suite can be found; for " + "use with non-interactive mode. If not given " + "non-interactive mode will try to take " + "this from the environment ($USER).", + ) - parser.add_argument('-N', "--new-kgo-dir", - help="specifies the name of the new KGO " - "subdirectory; for use with non-interactive " - "mode. If not given the KGO files will be " - "installed directly to the location currently " - "specified by the suite." - ) + parser.add_argument( + "-N", + "--new-kgo-dir", + help="specifies the name of the new KGO " + "subdirectory; for use with non-interactive " + "mode. If not given the KGO files will be " + "installed directly to the location currently " + "specified by the suite.", + ) - parser.add_argument('-P', "--platform", - help="specifies the platform this script is " - "running on. Defaults as ''. If the site is meto then" - "the platform will be autopopulated.") + parser.add_argument( + "-P", + "--platform", + help="specifies the platform this script is " + "running on. Defaults as ''. If the site is meto then" + "the platform will be autopopulated.", + ) - parser.add_argument('-E', "--extension", default=".rc", - help="The extension of the variables file, either .rc " - "or .cylc") + parser.add_argument( + "-E", + "--extension", + default=".rc", + help="The extension of the variables file, either .rc " "or .cylc", + ) args = parser.parse_args() @@ -498,10 +520,12 @@ def _split_lines(self, text, width): variables_extension = args.extension if suite_name is None or suite_user is None: - message = "'kgo_update.py' will no longer ask for your suite "\ - "information\n.To pass this into the script please use "\ - "the command line arguments.\nThese are documented at "\ - "the top of the file." + message = ( + "'kgo_update.py' will no longer ask for your suite " + "information\n.To pass this into the script please use " + "the command line arguments.\nThese are documented at " + "the top of the file." + ) sys.exit(message) # Prompt the user for the key options @@ -516,17 +540,19 @@ def _split_lines(self, text, width): # Get the site being run at site = get_site() - if site == 'meto' and new_kgo_dir == "install_in_place": - message = "Site meto should not use the 'install_in_place' option. "\ - "Check --new-kgo-dir is specified on the command line." \ - "Do you want to continue with the update?" + if site == "meto" and new_kgo_dir == "install_in_place": + message = ( + "Site meto should not use the 'install_in_place' option. " + "Check --new-kgo-dir is specified on the command line." + "Do you want to continue with the update?" + ) if not confirm(message, False): sys.exit() # Populate platform if not provided and at meto - if platform is None and site == 'meto': + if platform is None and site == "meto": hostname = os.uname()[1] - if hostname == 'uan01': + if hostname == "uan01": platform = "ex1a" elif hostname.startswith("xc"): platform = "xc40" @@ -553,10 +579,11 @@ def _split_lines(self, text, width): # Create a file to hold the update script script_path = os.path.expanduser("~/kgo_update_{0}.sh".format(new_kgo_dir)) if os.path.exists(script_path): - print("WARNING: Script file for this update already exists at {0} " - "and will be overwritten".format(script_path)) - if not confirm("Okay to overwrite script file (y/n)? ", - skip=confirm_skip): + print( + "WARNING: Script file for this update already exists at {0} " + "and will be overwritten".format(script_path) + ) + if not confirm("Okay to overwrite script file (y/n)? ", skip=confirm_skip): sys.exit("Aborting...") # Write the script file @@ -578,65 +605,76 @@ def _split_lines(self, text, width): site, platform, variables_extension, - skip=confirm_skip + skip=confirm_skip, ) # Open the file for viewing in user's editor if interactive: - print(f"\n\nOpening {script_path}\nHit Return to Step through, " - "q to print all\n\n") + print( + f"\n\nOpening {script_path}\nHit Return to Step through, " + "q to print all\n\n" + ) with open(script_path, "r") as f: l = 0 print_all = False for line in f: - print(line, end='') + print(line, end="") l += 1 if l == 10 and not print_all: l = 0 key = input() - if key.lower() == 'q': + if key.lower() == "q": print_all = True - print("\n\nPlease carefully check the commands in the above file " - "before continuing.\nIf changes need to be made then open a " - "new terminal and edit the file there.") + print( + "\n\nPlease carefully check the commands in the above file " + "before continuing.\nIf changes need to be made then open a " + "new terminal and edit the file there." + ) # Final confirmation before running print("WARNING: Once launched the script will make permanent changes") - if not confirm("Are you sure you want to continue (y/n)? ", - skip=confirm_skip): + if not confirm("Are you sure you want to continue (y/n)? ", skip=confirm_skip): sys.exit("Aborting...") - print(banner("Running KGO Update commands from {0}".format(script_path)), - flush=True) + print( + banner("Running KGO Update commands from {0}".format(script_path)), + flush=True, + ) - process = subprocess.Popen(["bash", script_path], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + process = subprocess.Popen( + ["bash", script_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) while True: line = process.stdout.readline() if not line and process.poll() is not None: break - print(line.decode(), end='', flush=True) + print(line.decode(), end="", flush=True) retcode = process.returncode if retcode != 0: - sys.exit(f"Script did not run successfully, update failed\n" - f"{process.stderr.read().decode()}\n") + sys.exit( + f"Script did not run successfully, update failed\n" + f"{process.stderr.read().decode()}\n" + ) # Print a final message print(banner("Generated KGO Script has run successfully")) - if site != 'meto': - print("\nThe script has created a copy of the variables.rc in your " + if site != "meto": + print( + "\nThe script has created a copy of the variables.rc in your " "$HOME directory; you should merge this with the one in your " - "working copy using xxdiff to update the KGO variables") + "working copy using xxdiff to update the KGO variables" + ) else: - print(f"\nThe kgo update for {platform} is complete.\n\nIf this was " - "run from 'meto_update_kgo.sh' in the UM, the generated files " - "will be moved to UMDIR on spice and the process will continue " - "for other platforms requested.\n\nOtherwise, eg. for " - "lfric_inputs, you will need to merge the generated variables " - "file with the one in your working copy and run the script " - "again on the next platform.") + print( + f"\nThe kgo update for {platform} is complete.\n\nIf this was " + "run from 'meto_update_kgo.sh' in the UM, the generated files " + "will be moved to UMDIR on spice and the process will continue " + "for other platforms requested.\n\nOtherwise, eg. for " + "lfric_inputs, you will need to merge the generated variables " + "file with the one in your working copy and run the script " + "again on the next platform." + ) if __name__ == "__main__": diff --git a/kgo_updates/kgo_update/meto_run_kgo_script.sh b/kgo_updates/kgo_update/meto_run_kgo_script.sh index bfce8b5c..0e16dd5a 100755 --- a/kgo_updates/kgo_update/meto_run_kgo_script.sh +++ b/kgo_updates/kgo_update/meto_run_kgo_script.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # *****************************COPYRIGHT******************************* # (C) Crown copyright Met Office. All rights reserved. # For further details please refer to the file COPYRIGHT.txt @@ -14,6 +14,8 @@ # rsyncs with the xcs. # This script is NOT intended to be run independently of '../meto_update_kgo.sh' +# shellcheck disable=SC2059 + # Set colour codes RED='\033[0;31m' GREEN='\033[0;32m' @@ -31,12 +33,13 @@ do P) platforms=${OPTARG};; F) variables_extension=${OPTARG};; V) version_number=${OPTARG};; + *) echo "Invalid option: -$flag" >&2; exit 1;; esac done # Create command to run python script kgo_command="./kgo_update.py" -if [ $new_release -eq 1 ]; then +if [[ $new_release -eq 1 ]]; then kgo_command="${kgo_command} --new-release" fi kgo_command="${kgo_command} -S $suite_name -N $new_kgo_dir -E $variables_extension" @@ -47,25 +50,25 @@ kgo_command_ex1a="$kgo_command -U $suite_user_ex1a -P ex1a" # Make a directory to store the script and variables file file in variables_dir=~/kgo_update_files/vn$version_number/$new_kgo_dir -mkdir -p $variables_dir +mkdir -p "$variables_dir" # If spice has kgo updates if [[ $platforms == *"spice"* ]] && [[ $platforms != *"azspice"* ]]; then printf "${GREEN}\n\nRunning KGO Update Script on spice.\n${NC}" # Run the Update Script - $kgo_command_spice - - if [[ $? -ne 0 ]]; then - printf "${RED}\nThe installation script has failed on spice.\n${NC}" - else + if $kgo_command_spice; then # Move the updated variables file and script into the ticket folder printf "${GREEN}\n\nSuccessfully installed on spice.\n${NC}" printf "${GREEN}Moving the generated files into spice ${variables_dir}.${NC}\n" - if [ $new_release -ne 1 ]; then - mv ~/variables${variables_extension}_${new_kgo_dir} ${variables_dir}/spice_updated_variables${variables_extension} + if [[ $new_release -ne 1 ]]; then + mv ~/"variables${variables_extension}_${new_kgo_dir}" \ + "${variables_dir}/spice_updated_variables${variables_extension}" fi - mv ~/kgo_update_${new_kgo_dir}.sh ${variables_dir}/spice_update_script.sh + mv ~/"kgo_update_${new_kgo_dir}.sh" \ + "${variables_dir}/spice_update_script.sh" + else + printf "${RED}\nThe installation script has failed on spice.\n${NC}" fi fi @@ -74,18 +77,18 @@ if [[ $platforms == *"azspice"* ]]; then printf "${GREEN}\n\nRunning KGO Update Script on azspice.\n${NC}" # Run the Update Script - $kgo_command_azspice - - if [[ $? -ne 0 ]]; then - printf "${RED}\nThe installation script has failed on azspice.\n${NC}" - else + if $kgo_command_azspice; then # Move the updated variables file and script into the ticket folder printf "${GREEN}\n\nSuccessfully installed on azspice.\n${NC}" printf "${GREEN}Moving the generated files into azspice ${variables_dir}.${NC}\n" - if [ $new_release -ne 1 ]; then - mv ~/variables${variables_extension}_${new_kgo_dir} ${variables_dir}/azspice_updated_variables${variables_extension} + if [[ $new_release -ne 1 ]]; then + mv ~/"variables${variables_extension}_${new_kgo_dir}" \ + "${variables_dir}/azspice_updated_variables${variables_extension}" fi - mv ~/kgo_update_${new_kgo_dir}.sh ${variables_dir}/azspice_update_script.sh + mv ~/"kgo_update_${new_kgo_dir}.sh" \ + "${variables_dir}/azspice_update_script.sh" + else + printf "${RED}\nThe installation script has failed on azspice.\n${NC}" fi fi @@ -96,34 +99,32 @@ if [[ $platforms == *"xc40"* ]]; then host_xc40=$(rose host-select xc) # SCP the python script to the xc40 - scp -q kgo_update.py frum@$host_xc40:~ + scp -q kgo_update.py "frum@${host_xc40}":~ # Define the commands to run on xc40 command=". /etc/profile ; module load scitools ; ${kgo_command_xc40}" # SSH to the xc40 with the run command - ssh -Y $host_xc40 $command - - if [[ $? -ne 0 ]]; then - printf "${RED}\nThe installation script has failed on xc40.\n${NC}" - else + if ssh -Y "$host_xc40" "$command"; then # rsync the generated variables file and script back to frum on linux # This cleans up the original files printf "${GREEN}\n\nSuccessfully installed on xc40.\n${NC}" printf "${GREEN}Rsyncing the generated files into spice ${variables_dir}.\n${NC}" - if [ $new_release -ne 1 ]; then + if [[ $new_release -ne 1 ]]; then rsync --remove-source-files -avz \ - frum@$host_xc40:~/variables${variables_extension}_${new_kgo_dir} \ - ${variables_dir}/xc40_updated_variables${variables_extension} + "frum@$host_xc40:~/variables${variables_extension}_${new_kgo_dir}" \ + "${variables_dir}/xc40_updated_variables${variables_extension}" fi rsync --remove-source-files -avz \ - frum@$host_xc40:~/kgo_update_${new_kgo_dir}.sh \ - ${variables_dir}/xc40_update_script.sh + "frum@${host_xc40}:~/kgo_update_${new_kgo_dir}.sh" \ + "${variables_dir}/xc40_update_script.sh" + else + printf "${RED}\nThe installation script has failed on xc40.\n${NC}" fi # Clean up kgo_update.py on xc40 - ssh -Yq $host_xc40 "rm kgo_update.py" + ssh -Yq "$host_xc40" "rm kgo_update.py" fi # If ex1a has kgo updates @@ -133,32 +134,30 @@ if [[ $platforms == *"ex1a"* ]]; then host_ex=$(rose host-select exab) # SCP the python script to the ex1a - scp -q kgo_update.py umadmin@$host_ex:~ + scp -q kgo_update.py "umadmin@$host_ex":~ # Define the commands to run on ex1a command="module load scitools ; ${kgo_command_ex1a}" # SSH to the ex1a with the run command - ssh -Y $host_ex $command - - if [[ $? -ne 0 ]]; then - printf "${RED}\nThe installation script has failed on ex1a.\n${NC}" - else + if ssh -Y "$host_ex" "$command"; then # rsync the generated variables file and script back to frum on linux # This cleans up the original files printf "${GREEN}\n\nSuccessfully installed on ex1a.\n${NC}" printf "${GREEN}Rsyncing the generated files into azspice ${variables_dir}.\n${NC}" - if [ $new_release -ne 1 ]; then + if [[ $new_release -ne 1 ]]; then rsync --remove-source-files -avz \ - umadmin@$host_ex:~/variables${variables_extension}_${new_kgo_dir} \ - ${variables_dir}/ex1a_updated_variables${variables_extension} + "umadmin@$host_ex:~/variables${variables_extension}_${new_kgo_dir}" \ + "${variables_dir}/ex1a_updated_variables${variables_extension}" fi rsync --remove-source-files -avz \ - umadmin@$host_ex:~/kgo_update_${new_kgo_dir}.sh \ - ${variables_dir}/ex1a_update_script.sh + "umadmin@$host_ex:~/kgo_update_${new_kgo_dir}.sh" \ + "${variables_dir}/ex1a_update_script.sh" + else + printf "${RED}\nThe installation script has failed on ex1a.\n${NC}" fi # Clean up kgo_update.py on ex1a - ssh -Yq $host_ex "rm kgo_update.py" + ssh -Yq "$host_ex" "rm kgo_update.py" fi diff --git a/kgo_updates/meto_update_kgo.sh b/kgo_updates/meto_update_kgo.sh index b7ec07d9..64e6b3eb 100755 --- a/kgo_updates/meto_update_kgo.sh +++ b/kgo_updates/meto_update_kgo.sh @@ -11,13 +11,15 @@ # The kgo_update.py script can be run with the --new-release option by providing # 'new-release' as a command line option to this script +# shellcheck disable=SC2059 + # Set colour codes RED='\033[0;31m' GREEN='\033[0;32m' NC='\033[0m' # No Color # Move to the location of the script -script_loc="$(dirname "$0")" +script_loc="$(dirname "$(realpath "$0")")" # Work out if we're running from azspice or old spice if [[ $HOSTNAME == "caz"* ]]; then @@ -25,11 +27,10 @@ if [[ $HOSTNAME == "caz"* ]]; then root_home="/home/users/umadmin" launch_platform=azspice # Check you can sudo in as umadmin - sudo -iu ${root_user} bash -c "echo ''" - if [[ $? -ne 0 ]]; then + sudo -iu ${root_user} bash -c "echo ''" || { printf "${RED} You were unable to run commands as umadmin - this is required to run this script" printf "This may be because of a password typo or similar" - fi + } else root_user="frum" root_home="/home/h01/frum" @@ -45,7 +46,7 @@ if [ $# -ne 0 ]; then else printf "${RED}'%s' is not a recognised command line argument.\n" "${1}" printf "The only command line option available is --new-release.\n" - read -p "Would you like to run in new-release mode (default n)? " answer + read -rp "Would you like to run in new-release mode (default n)? " answer answer=${answer:-"n"} if [[ $answer == "y" ]]; then new_release=1 @@ -58,14 +59,14 @@ fi # Prompt user for Update Details echo "Enter the platforms requiring a kgo update" echo "Enter platforms lowercase and space separated, eg. spice xc40 ex1a azspice" -read platforms +read -r platforms if [[ $platforms == *"spice"* ]] && [[ $platforms != *"azspice"* ]] || [[ $platforms == *"xc40"* ]]; then # Check we're not trying to install to spice while on azspice if [[ $launch_platform == "azspice" ]] && [[ $platforms == *"spice"* ]] && [[ $platforms != *"azspice"* ]]; then printf "${RED}Attempting to install spice kgo from azspice - this isn't possible" exit 1 fi - read -p "spice/xc40 Suite Username: " suite_user + read -rp "spice/xc40 Suite Username: " suite_user else suite_user=None fi @@ -75,26 +76,26 @@ if [[ $platforms == *"ex1a"* ]] || [[ $platforms == *"azspice"* ]]; then printf "${RED}Attempting to install azspice kgo from spice - this isn't possible" exit 1 fi - read -p "ex1a/azspice Suite Username: " suite_user_ex1a + read -rp "ex1a/azspice Suite Username: " suite_user_ex1a else suite_user_ex1a=None fi -read -p "Suite Name: " suite_name -read -p "Enter the path to the merged trunk WC (top directory): " wc_path +read -rp "Suite Name: " suite_name +read -rp "Enter the path to the merged trunk WC (top directory): " wc_path # Trim any trailing / from the end of the path wc_path=${wc_path%/} if [ ! -d "${wc_path}" ]; then printf "${RED}${wc_path} is not a valid path${NC}\n" exit 1 fi -read -p "Version Number (VV.V): " version_number -read -p "Ticket Number (TTTT): " ticket_number -read -p "How should the new kgo directory be named (default vn${version_number}_t${ticket_number}): " new_kgo_dir +read -rp "Version Number (VV.V): " version_number +read -rp "Ticket Number (TTTT): " ticket_number +read -rp "How should the new kgo directory be named (default vn${version_number}_t${ticket_number}): " new_kgo_dir new_kgo_dir=${new_kgo_dir:-"vn${version_number}_t${ticket_number}"} # Check in the working_copy rose-stem for .rc or .cylc files # Need this for the variables file extension -# Can't use Cylc version as .rc can be used in compatability mode +# Can't use Cylc version as .rc can be used in compatibility mode if [ -f "${wc_path}/rose-stem/suite.rc" ]; then variables_extension=".rc" elif [ -f "${wc_path}/rose-stem/suite.cylc" ] || [ -f "${wc_path}/rose-stem/flow.cylc" ]; then @@ -121,17 +122,17 @@ echo "New KGO Dir: ${new_kgo_dir}" if [ $new_release -eq 1 ]; then printf "${RED}WARNING: Running with --new-release enabled${NC}\n" fi -read -p "Run with the above settings y/n (default n): " run_script +read -rp "Run with the above settings y/n (default n): " run_script run_script=${run_script:-n} -if [ $run_script != "y" ]; then +if [[ $run_script != "y" ]]; then exit 0 fi # Move the kgo_update directory to frum on linux if [[ $launch_platform == "spice" ]]; then - scp -rq $script_loc/kgo_update ${root_user}@localhost:~ + scp -rq "${script_loc}"/kgo_update ${root_user}@localhost:~ else - sudo -iu ${root_user} bash -c "cp -r $script_loc/kgo_update ${root_home}" + sudo -iu ${root_user} bash -c "cp -r ${script_loc}/kgo_update ${root_home}" fi # Define command to run as frum @@ -148,13 +149,13 @@ command=". /etc/profile ; module load scitools ; cd kgo_update ; # Run the command as admin user if [[ $launch_platform == "spice" ]]; then - ssh -Y ${root_user}@localhost $command + ssh -Y ${root_user}@localhost "$command" else sudo -iu ${root_user} bash -c "cd $UMDIR ; $command" fi # Error Checking and rsyncing -variables_dir=kgo_update_files/vn${version_number}/${new_kgo_dir} +variables_dir="kgo_update_files/vn${version_number}/${new_kgo_dir}" succeeded_spice=0 succeeded_azspice=0 succeeded_xc40=0 @@ -166,13 +167,12 @@ if [[ $platforms == *"spice"* ]] && [[ $platforms != *"azspice"* ]]; then succeeded_spice=1 if [[ $new_release -ne 1 ]]; then printf "${GREEN}\n\nCopying the spice variables file into this working copy.\n${NC}" - scp -q ${root_user}@localhost:~/${variables_dir}/spice_updated_variables${variables_extension} \ - ${wc_path}/rose-stem/site/meto/variables_spice${variables_extension} - if [[ $? -ne 0 ]]; then + scp -q ${root_user}@localhost:~/"${variables_dir}/spice_updated_variables${variables_extension}" \ + "${wc_path}/rose-stem/site/meto/variables_spice${variables_extension}" || { printf "${RED}The copy of the spice variables file into this working copy has failed.\n${NC}" succeeded_spice=0 succeeded_all=0 - fi + } fi else succeeded_all=0 @@ -184,16 +184,14 @@ if [[ $platforms == *"azspice"* ]]; then succeeded_azspice=1 if [[ $new_release -ne 1 ]]; then printf "${GREEN}\n\nCopying the azspice variables file into this working copy.\n${NC}" - cp ${root_home}/${variables_dir}/azspice_updated_variables${variables_extension} \ - ${wc_path}/rose-stem/site/meto/variables_azspice${variables_extension} - if [[ $? -ne 0 ]]; then + cp "${root_home}/${variables_dir}/azspice_updated_variables${variables_extension}" \ + "${wc_path}/rose-stem/site/meto/variables_azspice${variables_extension}" || { printf "${RED}The copy of the azspice variables file into this working copy has failed.\n${NC}" succeeded_azspice=0 succeeded_all=0 - fi + } fi else - echo $file succeeded_all=0 fi fi @@ -204,17 +202,20 @@ if [[ $platforms == *"xc40"* ]]; then if [[ $new_release -ne 1 ]]; then printf "${GREEN}\n\nCopying the xc40 variables file into this working copy.\n${NC}" if [[ $launch_platform == "spice" ]]; then - scp -q ${root_user}@localhost:~/${variables_dir}/xc40_updated_variables${variables_extension} \ - ${wc_path}/rose-stem/site/meto/variables_xc40${variables_extension} + scp -q "${root_user}@localhost:~/${variables_dir}/xc40_updated_variables${variables_extension}" \ + "${wc_path}/rose-stem/site/meto/variables_xc40${variables_extension}" + rc=$? else - cp ${root_home}/${variables_dir}/xc40_updated_variables${variables_extension} \ - ${wc_path}/rose-stem/site/meto/variables_xc40${variables_extension} + cp "${root_home}/${variables_dir}/xc40_updated_variables${variables_extension}" \ + "${wc_path}/rose-stem/site/meto/variables_xc40${variables_extension}" + rc=$? fi - if [[ $? -ne 0 ]]; then + if [[ $rc -ne 0 ]]; then printf "${RED}The copy of the xc40 variables file into this working copy has failed.\n${NC}" succeeded_xc40=0 succeeded_all=0 fi + rc= fi else succeeded_all=0 @@ -227,17 +228,20 @@ if [[ $platforms == *"ex1a"* ]]; then if [[ $new_release -ne 1 ]]; then printf "${GREEN}\n\nCopying the ex1a variables file into this working copy.\n${NC}" if [[ $launch_platform == "spice" ]]; then - scp -q ${root_user}@localhost:~/${variables_dir}/ex1a_updated_variables${variables_extension} \ - ${wc_path}/rose-stem/site/meto/variables_ex1a${variables_extension} + scp -q "${root_user}@localhost:~/${variables_dir}/ex1a_updated_variables${variables_extension}" \ + "${wc_path}/rose-stem/site/meto/variables_ex1a${variables_extension}" + rc=$? else - cp ${root_home}/${variables_dir}/ex1a_updated_variables${variables_extension} \ - ${wc_path}/rose-stem/site/meto/variables_ex1a${variables_extension} + cp "${root_home}/${variables_dir}/ex1a_updated_variables${variables_extension}" \ + "${wc_path}/rose-stem/site/meto/variables_ex1a${variables_extension}" + rc=$? fi - if [[ $? -ne 0 ]]; then + if [[ $rc -ne 0 ]]; then printf "${RED}The copy of the ex1a variables file into this working copy has failed.\n${NC}" succeeded_ex1a=0 succeeded_all=0 fi + rc= fi else succeeded_all=0 @@ -251,7 +255,7 @@ else fi if [[ $platforms == *"xc40"* ]] || [[ $platforms == *"ex1a"* ]]; then - read -p "Enter 1 to rsync UM KGO, 2 to rsync lfricinputs KGO (default 1): " rsync_type + read -rp "Enter 1 to rsync UM KGO, 2 to rsync lfricinputs KGO (default 1): " rsync_type if [[ $rsync_type == "2" ]]; then rsync_dir="lfricinputs/kgo/" else @@ -265,15 +269,18 @@ if [[ $succeeded_xc40 -eq 1 ]]; then host_rsync=$(rose host-select xc) rsync_com="ssh -Y ${host_rsync} 'rsync -av /projects/um1/standard_jobs/${rsync_dir} xcslr0:/common/um1/standard_jobs/${rsync_dir}'" if [[ $launch_platform == "spice" ]]; then - ssh -Y ${root_user}@localhost $rsync_com + ssh -Y ${root_user}@localhost "$rsync_com" + rc=$? else sudo -iu ${root_user} bash -c "$rsync_com" + rc=$? fi - if [[ $? -ne 0 ]]; then + if [[ $rc -ne 0 ]]; then printf "${RED}The rsync to the xcs has failed.\n${NC}" else - printf "${Green}The rsync to the xcs has succeeded.\n${NC}" + printf "${GREEN}The rsync to the xcs has succeeded.\n${NC}" fi + rc= elif [[ $platforms == *"xc40"* ]]; then printf "${RED}\n\nSkipping the rsync to the xcs as the xc40 install failed.\n${NC}" fi @@ -288,29 +295,35 @@ if [[ $succeeded_ex1a -eq 1 ]]; then # rsync to EXZ rsync_com="ssh -Y ${host_rsync} 'rsync -av /common/internal/umdir/standard_jobs/${rsync_dir} login.exz:/common/umdir/standard_jobs/${rsync_dir}'" if [[ $launch_platform == "spice" ]]; then - ssh -Y ${root_user}@localhost $rsync_com + ssh -Y ${root_user}@localhost "$rsync_com" + rc=$? else sudo -iu ${root_user} bash -c "$rsync_com" + rc=$? fi - if [[ $? -ne 0 ]]; then + if [[ $rc -ne 0 ]]; then printf "${RED}The rsync to the exz has failed.\n${NC}" else - printf "${Green}The rsync to the exz has succeeded.\n${NC}" + printf "${GREEN}The rsync to the exz has succeeded.\n${NC}" fi + rc= # rsync to EXCD excd_host=$(rose host-select excd) rsync_com="ssh -Y ${host_rsync} 'rsync -av /common/internal/umdir/standard_jobs/${rsync_dir} ${excd_host}:/common/internal/umdir/standard_jobs/${rsync_dir}'" if [[ $launch_platform == "spice" ]]; then - ssh -Y ${root_user}@localhost $rsync_com + ssh -Y ${root_user}@localhost "$rsync_com" + rc=$? else sudo -iu ${root_user} bash -c "$rsync_com" + rc=$? fi - if [[ $? -ne 0 ]]; then + if [[ $rc -ne 0 ]]; then printf "${RED}The rsync to the excd has failed.\n${NC}" else - printf "${Green}The rsync to the excd has succeeded.\n${NC}" + printf "${GREEN}The rsync to the excd has succeeded.\n${NC}" fi + rc= elif [[ $platforms == *"ex1a"* ]]; then printf "${RED}\n\nSkipping the rsync to the exz/cd as the exab install failed.\n${NC}" fi @@ -343,4 +356,4 @@ if [[ $platforms == *"ex1a"* ]]; then else printf "${RED}Installation on ex1a unsuccessful. Review output for error.\n${NC}" fi -fi \ No newline at end of file +fi diff --git a/lfric_macros/apply_macros.py b/lfric_macros/apply_macros.py index 63ddf627..7c56e452 100755 --- a/lfric_macros/apply_macros.py +++ b/lfric_macros/apply_macros.py @@ -10,13 +10,13 @@ Warning: Should only be run on a Test branch or by CR on commit to trunk """ +import argparse +import ast import os import re -import ast import shutil -import argparse -import tempfile import subprocess +import tempfile BLACK_COMMAND = "black --line-length=80" CLASS_NAME_REGEX = r"vn\d+(_t\d+\w*)?" @@ -544,7 +544,7 @@ def find_last_macro(self, macros, meta_dir): after_tag = re.search(rf"AFTER_TAG{TAG_REGEX}", macro).group(1) except AttributeError as exc: raise Exception( - "Couldn't find an after tag in the macro:\n" f"{macro}" + f"Couldn't find an after tag in the macro:\n{macro}" ) from exc found_macro = macro macros.remove(found_macro) diff --git a/lfric_macros/files/template_versions.py b/lfric_macros/files/template_versions.py index 098890e1..fca00431 100644 --- a/lfric_macros/files/template_versions.py +++ b/lfric_macros/files/template_versions.py @@ -2,6 +2,7 @@ from metomi.rose.upgrade import MacroUpgrade + class UpgradeError(Exception): """Exception created when an upgrade fails.""" diff --git a/lfric_macros/release_lfric.py b/lfric_macros/release_lfric.py index d1bcd013..c2fcb9b8 100755 --- a/lfric_macros/release_lfric.py +++ b/lfric_macros/release_lfric.py @@ -26,9 +26,9 @@ from apply_macros import ( ApplyMacros, apply_macros_main, + apply_styling, get_root_path, read_versions_file, - apply_styling, split_macros, version_number, ) @@ -72,7 +72,7 @@ def raise_exception(result, command): Raise an exception if a subprocess command has failed """ - raise Exception(f"[FAIL] Error running command: '{command}'\n" f"{result.stderr}") + raise Exception(f"[FAIL] Error running command: '{command}'\n{result.stderr}") def set_dependency_path(args): diff --git a/lfric_macros/tests/test_apply_macros.py b/lfric_macros/tests/test_apply_macros.py index 42ed52d7..25855332 100644 --- a/lfric_macros/tests/test_apply_macros.py +++ b/lfric_macros/tests/test_apply_macros.py @@ -1,6 +1,8 @@ -import pytest -import subprocess import shutil +import subprocess + +import pytest + from ..apply_macros import * # A macro that we want to find for these tests @@ -44,10 +46,7 @@ def __repr__(self): test_versions_file = [f"{x}\n" for x in test_versions_file.split("\n")] # The expected result from split_macros for the versions -expected_split_macros = [ - desired_macro, - existing_macro -] +expected_split_macros = [desired_macro, existing_macro] # ApplyMacros below requires an LFRic Apps working copy to work - check out the # head of the lfric_apps trunk for this purpose. The actual contents of the @@ -58,25 +57,19 @@ def __repr__(self): check=False, capture_output=True, text=True, - timeout=120 + timeout=120, ) if result.returncode: raise RuntimeError( "Failed to checkout required LFRic Apps Working Copy with error ", - result.stderr + result.stderr, ) # Create an instance of the apply_macros class # Pass a known directory in as the Jules and Core sources as these are not # required for testing -am = ApplyMacros( - "vn0.0_t001", - None, - None, - appsdir, - "/tmp", - "/tmp" -) +am = ApplyMacros("vn0.0_t001", None, None, appsdir, "/tmp", "/tmp") + def test_split_macros(): m = split_macros(test_versions_file) @@ -103,10 +96,10 @@ def test_parse_macro(): expected_dict = { "before_tag": "vn0.0_t000", "commands": ( - ' self.add_setting(\n' - ' config, ["namelist:namelist1", "opt1"], "value1"\n' - ' )\n' - ) + " self.add_setting(\n" + ' config, ["namelist:namelist1", "opt1"], "value1"\n' + " )\n" + ), } assert am.parsed_macros["meta_dir"] == expected_dict assert am.ticket_number == "#001" @@ -117,14 +110,19 @@ def test_parse_macro(): def test_read_meta_imports(): am.parsed_macros["tests/test_meta_dir"] = {} - am.parsed_macros["tests/test_meta_dir"]["imports"] = am.read_meta_imports("tests/test_meta_dir") + am.parsed_macros["tests/test_meta_dir"]["imports"] = am.read_meta_imports( + "tests/test_meta_dir" + ) expected_imports = [ os.path.join(am.root_path, "science", "gungho"), - os.path.join(am.root_path, "applications", "lfric_atm") + os.path.join(am.root_path, "applications", "lfric_atm"), ] assert am.parsed_macros["tests/test_meta_dir"]["imports"] == expected_imports expected_meta = [os.path.join(am.root_path, "applications", "lfric_atm")] - assert am.read_meta_imports("tests/test_meta_dir/rose-app.conf", "meta") == expected_meta + assert ( + am.read_meta_imports("tests/test_meta_dir/rose-app.conf", "meta") + == expected_meta + ) def test_determine_import_order(): @@ -136,22 +134,13 @@ def test_determine_import_order(): am.parsed_macros["import3"]["imports"] = ["import4"] am.parsed_macros["import4"]["imports"] = [] am.parsed_macros["import2"]["imports"] = [] - expected_order = [ - "import2", - "import4", - "import3", - "import1" - ] + expected_order = ["import2", "import4", "import3", "import1"] assert am.determine_import_order("import1") == expected_order def test_combine_macros(): - am.parsed_macros["importA"] = { - "commands": " importA command" - } - am.parsed_macros["importB"] = { - "commands": " importB command" - } + am.parsed_macros["importA"] = {"commands": " importA command"} + am.parsed_macros["importB"] = {"commands": " importB command"} expected_combined = ( " # Commands From: importA\n importA command\n" " # Commands From: importB\n importB command\n" @@ -167,9 +156,18 @@ def test_parse_application_section(): def test_deduplicate_list(): - assert deduplicate_list([1,2,3]) == [1,2,3] - assert deduplicate_list([1,2,2,3,3,3,]) == [1,2,3] - assert deduplicate_list([1,2,1,3,2]) == [1,2,3] + assert deduplicate_list([1, 2, 3]) == [1, 2, 3] + assert deduplicate_list( + [ + 1, + 2, + 2, + 3, + 3, + 3, + ] + ) == [1, 2, 3] + assert deduplicate_list([1, 2, 1, 3, 2]) == [1, 2, 3] def test_match_python_imports(): @@ -179,8 +177,9 @@ def test_match_python_imports(): assert match_python_import("import m as n") == True assert match_python_import("false") == False + # Remove appsdir -@pytest.fixture(scope='session', autouse=True) +@pytest.fixture(scope="session", autouse=True) def remove_tempdir(): yield shutil.rmtree(appsdir) diff --git a/lfric_styling.py b/lfric_styling.py old mode 100644 new mode 100755 index 86405e92..e010a996 --- a/lfric_styling.py +++ b/lfric_styling.py @@ -6,7 +6,7 @@ # Modified date: 14/03/2025 # *****************************COPYRIGHT******************************* -''' +""" Python script to apply LFRic styling to .F90/.f90 files, given a path/directory. Current implementation takes a valid path and traverses from the top level file @@ -15,13 +15,14 @@ Styling currently being applied: -lowercasing keywords -''' +""" -import re -import sys import argparse import os +import re +import sys from pathlib import Path + from styling_keywords import NEW_KEYWORDS @@ -30,12 +31,12 @@ def lowercase_keywords(file): Lowercase words in a file when they match a word in the keywords set. """ print("Lowercasing keywords in", file) - with open(file, 'r') as fp: + with open(file, "r") as fp: lines = fp.read() for keyword in NEW_KEYWORDS: pattern = rf"\b{re.escape(keyword.upper())}\b" lines = re.sub(pattern, convert_to_lower, lines) - with open(file, 'w') as fp: + with open(file, "w") as fp: for line in lines: fp.write(line) @@ -66,16 +67,17 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument( - "-d", "--directory", + "-d", + "--directory", metavar="home/user/etc", type=Path, default=Path(), - help="path to a directory of files." + help="path to a directory of files.", ) arguments = parser.parse_args() apply_styling(arguments.directory) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/nightly_testing/generate_test_suite_cron.py b/nightly_testing/generate_test_suite_cron.py index 9ef75fae..c822dfe9 100755 --- a/nightly_testing/generate_test_suite_cron.py +++ b/nightly_testing/generate_test_suite_cron.py @@ -32,12 +32,13 @@ works. """ +import argparse import os -import sys import re -import yaml -import argparse import subprocess +import sys + +import yaml DEFAULT_CYLC_VERSION = "8" DEPENDENCIES = { @@ -72,6 +73,7 @@ DATE_BASE = "date +\\%Y-\\%m-\\%d" MONITORING_TIME = "00 06" + def run_command(command): """ Run a subprocess command and return the result object @@ -335,7 +337,7 @@ def generate_main_job(name, suite, log_file, wc_path, cylc_version): def_link = os.path.join(CYLC_INSTALL, "cylc-8") cron_job += ( f'[ "$(readlink -- "{next_link}")" != "$(readlink -- "{def_link}")" ] ' - f'&& ({job_command})' + f"&& ({job_command})" ) else: cron_job += job_command @@ -394,7 +396,7 @@ def parse_cl_args(): "--cylc_path", default="~metomi", help="The location of the cylc installation required for testing `next-cylc`" - "configs." + "configs.", ) parser.add_argument( "--install", @@ -436,9 +438,7 @@ def parse_cl_args(): main_crontab += f"# {repo.upper()} SUITES\n" main_crontab += 80 * "#" + 2 * "\n" last_repo = repo - main_crontab += generate_cron_job( - suite_name, suites[suite_name], args.cron_log - ) + main_crontab += generate_cron_job(suite_name, suites[suite_name], args.cron_log) main_crontab += 3 * "\n" with open(args.cron_file, "w") as outfile: diff --git a/nightly_testing/retrigger_nightlies.py b/nightly_testing/retrigger_nightlies.py index b4096a83..387edfb6 100755 --- a/nightly_testing/retrigger_nightlies.py +++ b/nightly_testing/retrigger_nightlies.py @@ -17,12 +17,12 @@ """ import os -import sys import re -import subprocess import sqlite3 -from time import sleep +import subprocess +import sys from datetime import datetime, timedelta +from time import sleep def run_command(command): @@ -68,8 +68,7 @@ def check_for_failed_tasks(conn): "SELECT name, status FROM task_states WHERE status LIKE '%failed%'" ).fetchall() res_subfail = conn.execute( - "SELECT name, status FROM task_states " - "WHERE status LIKE '%submit-failed%'" + "SELECT name, status FROM task_states WHERE status LIKE '%submit-failed%'" ).fetchall() return res_failed + res_subfail @@ -82,7 +81,7 @@ def ask_yn(message): rval = "" while rval.lower().strip() not in ["y", "n"]: rval = input(f"{message}? (y/n) ") - return rval.lower().strip()=="y" + return rval.lower().strip() == "y" def restart_suite(suite): diff --git a/nightly_testing/tests/test_generate_test_suite_cron.py b/nightly_testing/tests/test_generate_test_suite_cron.py index fa0019a7..8d7cc1a1 100644 --- a/nightly_testing/tests/test_generate_test_suite_cron.py +++ b/nightly_testing/tests/test_generate_test_suite_cron.py @@ -8,17 +8,19 @@ ( ["um"], "scratch/dir/", - "fcm co -q --force fcm:um.xm_tr@HEAD scratch/dir/wc_um ; " + "fcm co -q --force fcm:um.xm_tr@HEAD scratch/dir/wc_um ; ", ), ( ["um", "lfric"], "scratch/dir", - "fcm co -q --force fcm:um.xm_tr@HEAD scratch/dir/wc_um ; fcm co -q --force fcm:lfric.xm_tr@HEAD scratch/dir/wc_lfric ; " - ) + "fcm co -q --force fcm:um.xm_tr@HEAD scratch/dir/wc_um ; fcm co -q --force fcm:lfric.xm_tr@HEAD scratch/dir/wc_lfric ; ", + ), ] + + @pytest.mark.parametrize( ("inlist", "scratch", "expected"), - [test_data for test_data in data_join_checkout_commands] + [test_data for test_data in data_join_checkout_commands], ) def test_join_checkout_commands(inlist, scratch, expected): assert join_checkout_commands(inlist, scratch) == expected @@ -31,9 +33,10 @@ def test_join_checkout_commands(inlist, scratch, expected): "cp -rf path/to/wc path/to/wc_heads ; sed -i -e 's/^\\(export .*_revision=@\\).*/\\1HEAD/' path/to/wc_heads/dependencies.sh ; sed -i -e 's/^\\(export .*_rev=\\).*/\\1HEAD/' path/to/wc_heads/dependencies.sh ; ", ) ] + + @pytest.mark.parametrize( - ("wc_path", "expected"), - [test_data for test_data in data_lfric_heads_sed] + ("wc_path", "expected"), [test_data for test_data in data_lfric_heads_sed] ) def test_lfric_heads_sed(wc_path, expected): assert lfric_heads_sed(wc_path) == (expected) @@ -41,55 +44,37 @@ def test_lfric_heads_sed(wc_path, expected): # Test generate_cron_timing_str data_generate_cron_timing_str = [ - ( - {"period": "weekly", "cron_launch": "30 00"}, - "main", - "30 00 * * 1 " - ), - ( - {"period": "nightly", "cron_launch": "30 00"}, - "main", - "30 00 * * 2-5 " - ), + ({"period": "weekly", "cron_launch": "30 00"}, "main", "30 00 * * 1 "), + ({"period": "nightly", "cron_launch": "30 00"}, "main", "30 00 * * 2-5 "), ( {"period": "nightly_all", "cron_launch": "30 00"}, "main", - "30 00 * * 1-5 " - ), - ( - {"period": "weekly", "cron_clean": "30 00"}, - "clean", - "30 00 * * 7 " - ), - ( - {"period": "nightly", "cron_clean": "30 00"}, - "clean", - "30 00 * * 3-6 " + "30 00 * * 1-5 ", ), + ({"period": "weekly", "cron_clean": "30 00"}, "clean", "30 00 * * 7 "), + ({"period": "nightly", "cron_clean": "30 00"}, "clean", "30 00 * * 3-6 "), ( {"period": "nightly_all", "cron_clean": "30 00"}, "clean", - "30 00 * * 2-6 " - ), - ( - {"period": "weekly", "cron_clean": "30 00"}, - "monitoring", - "00 06 * * 1 " + "30 00 * * 2-6 ", ), + ({"period": "weekly", "cron_clean": "30 00"}, "monitoring", "00 06 * * 1 "), ( {"period": "nightly", "cron_clean": "30 00"}, "monitoring", - "00 06 * * 2-5 " + "00 06 * * 2-5 ", ), ( {"period": "nightly_all", "cron_clean": "30 00"}, "monitoring", - "00 06 * * 1-5 " + "00 06 * * 1-5 ", ), ] + + @pytest.mark.parametrize( ("suite", "mode", "expected"), - [test_data for test_data in data_generate_cron_timing_str] + [test_data for test_data in data_generate_cron_timing_str], ) def test_generate_cron_timing_str(suite, mode, expected): assert generate_cron_timing_str(suite, mode) == expected @@ -101,24 +86,26 @@ def test_generate_cron_timing_str(suite, mode, expected): "7", "suite_name", "cron_log", - f"{PROFILE} ; export CYLC_VERSION=7 ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; rose suite-clean -y -q suite_name >> cron_log 2>&1\n" + f"{PROFILE} ; export CYLC_VERSION=7 ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; rose suite-clean -y -q suite_name >> cron_log 2>&1\n", ), ( "8", "suite_name", "cron_log", - f"{PROFILE} ; export CYLC_VERSION=8 ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; cylc clean --timeout=7200 -y -q suite_name >> cron_log 2>&1\n" + f"{PROFILE} ; export CYLC_VERSION=8 ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; cylc clean --timeout=7200 -y -q suite_name >> cron_log 2>&1\n", ), ( "8-next", "suite_name", "cron_log", - f"{PROFILE} ; export CYLC_VERSION=8-next ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; cylc clean --timeout=7200 -y -q suite_name >> cron_log 2>&1\n" - ) + f"{PROFILE} ; export CYLC_VERSION=8-next ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; cylc clean --timeout=7200 -y -q suite_name >> cron_log 2>&1\n", + ), ] + + @pytest.mark.parametrize( ("cylc_version", "name", "log_file", "expected"), - [test_data for test_data in data_generate_clean_commands] + [test_data for test_data in data_generate_clean_commands], ) def test_generate_clean_commands(cylc_version, name, log_file, expected): assert generate_clean_commands(cylc_version, name, log_file) == expected @@ -131,26 +118,28 @@ def test_generate_clean_commands(cylc_version, name, log_file, expected): "path/to/wc", "7", "suite_name", - "export CYLC_VERSION=7 ; rose stem --group=all --name=suite_name --source=path/to/wc " + "export CYLC_VERSION=7 ; rose stem --group=all --name=suite_name --source=path/to/wc ", ), ( {"groups": "nightly"}, "path/to/wc", "8", "suite_name", - "export CYLC_VERSION=8 ; rose stem --group=nightly --workflow-name=suite_name --source=path/to/wc " + "export CYLC_VERSION=8 ; rose stem --group=nightly --workflow-name=suite_name --source=path/to/wc ", ), ( {"groups": "nightly"}, "path/to/wc", "8-next", "suite_name", - "export CYLC_VERSION=8-next ; rose stem --group=nightly --workflow-name=suite_name --source=path/to/wc " - ) + "export CYLC_VERSION=8-next ; rose stem --group=nightly --workflow-name=suite_name --source=path/to/wc ", + ), ] + + @pytest.mark.parametrize( ("suite", "wc_path", "cylc_version", "name", "expected"), - [test_data for test_data in data_generate_rose_stem_command] + [test_data for test_data in data_generate_rose_stem_command], ) def test_generate_rose_stem_command(suite, wc_path, cylc_version, name, expected): assert generate_rose_stem_command(suite, wc_path, cylc_version, name) == expected @@ -163,39 +152,35 @@ def test_generate_rose_stem_command(suite, wc_path, cylc_version, name, expected "repo": "um", "revisions": "heads", }, - "--source=fcm:casim.xm_tr@HEAD --source=fcm:jules.xm_tr@HEAD --source=fcm:mule.xm_tr@HEAD --source=fcm:shumlib.xm_tr@HEAD --source=fcm:socrates.xm_tr@HEAD --source=fcm:ukca.xm_tr@HEAD " + "--source=fcm:casim.xm_tr@HEAD --source=fcm:jules.xm_tr@HEAD --source=fcm:mule.xm_tr@HEAD --source=fcm:shumlib.xm_tr@HEAD --source=fcm:socrates.xm_tr@HEAD --source=fcm:ukca.xm_tr@HEAD ", ), ( { "repo": "um", "revisions": "set", }, - "" + "", ), ( { "repo": "um", }, - "" + "", ), ( { "repo": "jules", "revisions": "heads", }, - "" + "", ), - ( - { - "repo": "lfric_apps", - "revisions": "heads" - }, - "" - ) + ({"repo": "lfric_apps", "revisions": "heads"}, ""), ] + + @pytest.mark.parametrize( ("suite", "expected"), - [test_data for test_data in data_populate_heads_sources] + [test_data for test_data in data_populate_heads_sources], ) def test_populate_heads_sources(suite, expected): assert populate_heads_sources(suite) == expected @@ -203,20 +188,14 @@ def test_populate_heads_sources(suite, expected): # Test populate_cl_variables data_populate_cl_variables = [ - ( - { - "vars": ["var1", "var2", "var3"] - }, - "-S var1 -S var2 -S var3 " - ), - ( - {}, - "" - ) + ({"vars": ["var1", "var2", "var3"]}, "-S var1 -S var2 -S var3 "), + ({}, ""), ] + + @pytest.mark.parametrize( ("suite", "expected"), - [test_data for test_data in data_populate_cl_variables] + [test_data for test_data in data_populate_cl_variables], ) def test_populate_cl_variables(suite, expected): assert populate_cl_variables(suite) == expected @@ -224,22 +203,15 @@ def test_populate_cl_variables(suite, expected): # Test major_cylc_version data_major_cylc_version = [ - ( - "7", - "7" - ), - ( - "8", - "8" - ), - ( - "8-next", - "8" - ), + ("7", "7"), + ("8", "8"), + ("8-next", "8"), ] + + @pytest.mark.parametrize( ("version", "expected"), - [test_data for test_data in data_major_cylc_version] + [test_data for test_data in data_major_cylc_version], ) def test_major_cylc_version(version, expected): assert major_cylc_version(version) == expected diff --git a/script_copyright_checker/bin/copyright_checker.py b/script_copyright_checker/bin/copyright_checker.py index 46517d32..54e986c5 100755 --- a/script_copyright_checker/bin/copyright_checker.py +++ b/script_copyright_checker/bin/copyright_checker.py @@ -8,21 +8,18 @@ Script which tests code files within the UM repository to ensure they contain a recognised copyright notice. """ -import re +import argparse import os -import sys -import subprocess +import re +from textwrap import wrap from fcm_bdiff import ( get_branch_diff_filenames, - text_decoder, - is_trunk, - use_mirror, get_branch_info, get_url, + is_trunk, + use_mirror, ) -import argparse -from textwrap import wrap # Desired maximum column width for output - we make an exception # for filenames, which are always printed on a single line to aid @@ -56,13 +53,9 @@ def load_templates(filter_pattern): template_path = "." if "CYLC_TASK_WORK_PATH" in os.environ: - template_path = os.path.join( - os.environ["CYLC_TASK_WORK_PATH"], "file", "" - ) + template_path = os.path.join(os.environ["CYLC_TASK_WORK_PATH"], "file", "") - template_files = files_to_process( - template_path, [], filter_pattern=filter_pattern - ) + template_files = files_to_process(template_path, [], filter_pattern=filter_pattern) for filename in template_files: with open(filename) as file: @@ -143,9 +136,7 @@ def main(inputs, ignore_list): regex_templates_raw.extend(load_templates(filter_pattern=filter_tmp)) for filename, template_lines in regex_templates_raw: - regex_templates.append( - (filename, re.compile(r"\n".join(template_lines))) - ) + regex_templates.append((filename, re.compile(r"\n".join(template_lines)))) files_to_check = [] for file_input in inputs: @@ -212,9 +203,7 @@ def parse_options(): action="store", dest="ignore", default=None, - help=( - "ignore filename/s containing (comma separated list of patterns)" - ), + help=("ignore filename/s containing (comma separated list of patterns)"), ) parser.add_argument( "--base_path", @@ -228,8 +217,7 @@ def parse_options(): action="store_true", default=False, help=( - "run on use the full file list when trunk, " - "else run on fcm branch-diff" + "run on use the full file list when trunk, " "else run on fcm branch-diff" ), ) excl_group.add_argument( diff --git a/script_umdp3_checker/bin/umdp3_check.pl b/script_umdp3_checker/bin/umdp3_check.pl index 0a41347e..70d65147 100755 --- a/script_umdp3_checker/bin/umdp3_check.pl +++ b/script_umdp3_checker/bin/umdp3_check.pl @@ -1010,13 +1010,13 @@ sub run_checks { } # read in data from file to $data, then - my $io_array = new IO::ScalarArray \@file_lines; + my $io_array = IO::ScalarArray->new(\@file_lines); my $mimetype = mimetype($io_array); # if we can't detect a mime type, try some tricks to aid detection if ( $mimetype =~ /text\/plain/sxm ) { my @mime_file_lines = grep !/^\s*\#/sxm, @file_lines; - $io_array = new IO::ScalarArray \@mime_file_lines; + $io_array = IO::ScalarArray->new(\@mime_file_lines); $mimetype = mimetype($io_array); } @@ -1133,12 +1133,12 @@ sub run_checks { $filename .= "."; } $filename = $log_cylc . "." . $filename . "report"; - my $fileres = open( FH, '>', $filename ); + my $fileres = open( my $fh, '>', $filename ); if ( !defined $fileres ) { die "ERR: $filename\n"; } - print FH $failure_text; - close(FH); + print $fh $failure_text; + close($fh); } } $message = ''; diff --git a/script_updater.sh b/script_updater.sh index 6c4de48a..3889a052 100755 --- a/script_updater.sh +++ b/script_updater.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # *****************************COPYRIGHT******************************* # (C) Crown copyright Met Office. All rights reserved. @@ -6,40 +6,20 @@ # which you should have received as part of this distribution. # *****************************COPYRIGHT******************************* -#set -eu +# set -eu +# Note (yp): if this is a standalone script, we should use set -e +# and avoid the || { exit $?; } construct below. git_repos="SimSys_Scripts.git" github_url="git@github.com:MetOffice/$git_repos" -git_branch="add_umdp3checker_script" -#git_branch="main" +git_branch="main" clone_destination="${CYLC_SUITE_SHARE_DIR}/imported_github_scripts" +echo "Remove ${clone_destination} if it exists" +rm -rf "$clone_destination" 1>/dev/null || { exit $?; } -echo "Git-scripts updater has started running" - -rm -rf "${CYLC_SUITE_SHARE_DIR}/imported_github_scripts" -if [[ $? != 0 ]]; then - echo "Couldn't remove specified folder. Try checking permissions" - echo " \"$clone_destination\"" - exit 1 - else - echo "Successfully removed old ${git_repos%.git} git directory" - echo " \"$clone_destination\"" -fi - -# Mildly circular dependency here... The name of the branch needs changing - -# once ths branch has been committed.. See commented out 'branch' name above. -git clone -b $git_branch --single-branch $github_url "$clone_destination" 2>&1 -if [[ $? != 0 ]]; then - echo -e "\nUnable to clone remote git repo into specified location." - echo " Check git branch, git url, git access as well as" - echo -e " destination path, and permissions\n" - echo "GitHub url = \"$github_url\"" - echo "git branch = \"$git_branch\"" - echo "destination path = \"$clone_destination\"" - exit 1 - else - echo -e "\nGit repo successfully cloned\n" -fi - -echo -e "\nGithub scripts updated\n" +echo "Update ${git_repos%.git} in ${clone_destination}" +set -x +git clone -q -b $git_branch --single-branch $github_url "$clone_destination" \ + || { exit $?; } +{ set +x; } 2>/dev/null diff --git a/styling.py b/styling.py index 6975e20c..c06af63e 100755 --- a/styling.py +++ b/styling.py @@ -6,7 +6,7 @@ # Created date: 24/10/2023 # Modified date: 14/03/2025 # *****************************COPYRIGHT******************************* -''' +""" ## NOTE ## This module is one of several for which the Master copy is in the UM repository. When making changes, please ensure the changes are made in the UM @@ -15,17 +15,23 @@ This module contains various functions for applying UMDP3 styling to Fortran source code -''' +""" import re import sys + from fstring_parse import * -from styling_keywords import KEYWORDS, CODE_REPLACEMENTS, COMMENT_REPLACEMENTS, FORTRAN_TYPES +from styling_keywords import ( + CODE_REPLACEMENTS, + COMMENT_REPLACEMENTS, + FORTRAN_TYPES, + KEYWORDS, +) def replace_patterns(line, str_continuation): - '''Replace various patterns according to the styling guidelines on - the provided line, returning the result''' + """Replace various patterns according to the styling guidelines on + the provided line, returning the result""" if len(line.strip()) == 0: return line @@ -35,11 +41,11 @@ def replace_patterns(line, str_continuation): stripline = workline.strip() # Pre-processor lines start with #. Ignore them completely. - pre_proc = (stripline[0] == "#") + pre_proc = stripline[0] == "#" # Lines that are completely commented start with a bang and are also # ignored completely. - all_comment = (stripline[0] == "!") + all_comment = stripline[0] == "!" if pre_proc or all_comment: return line @@ -65,14 +71,14 @@ def replace_patterns(line, str_continuation): for pattern, replacement in CODE_REPLACEMENTS: m = re.search(pattern, match_line, flags=re.IGNORECASE) if m: - for n in range(len(m.groups())+1): - replacement = re.sub(r'(\\{0:s}|\\g<{0:s}>)'.format(str(n)), - line[m.start(n):m.end(n)], - replacement) + for n in range(len(m.groups()) + 1): + replacement = re.sub( + r"(\\{0:s}|\\g<{0:s}>)".format(str(n)), + line[m.start(n) : m.end(n)], + replacement, + ) - line = "".join([line[:m.start(0)], - replacement, - line[m.end(0):]]) + line = "".join([line[: m.start(0)], replacement, line[m.end(0) :]]) workline = clean_str_continuation(line, str_continuation) @@ -86,8 +92,7 @@ def replace_patterns(line, str_continuation): blank_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:" \ - "".format(e.msg)) + print("{0:s} Line simplification has failed for:" "".format(e.msg)) print(line) exit(1) @@ -98,8 +103,8 @@ def replace_patterns(line, str_continuation): def replace_comment_patterns(line, str_continuation): - '''Replace various patterns according to the styling guidelines on - the provided line, returning the result''' + """Replace various patterns according to the styling guidelines on + the provided line, returning the result""" if len(line.strip()) == 0: return line @@ -109,7 +114,7 @@ def replace_comment_patterns(line, str_continuation): stripline = workline.strip() # Pre-processor lines start with #. Ignore them completely. - pre_proc = (stripline[0] == "#") + pre_proc = stripline[0] == "#" if pre_proc: return line @@ -132,14 +137,14 @@ def replace_comment_patterns(line, str_continuation): for pattern, replacement in COMMENT_REPLACEMENTS: m = re.search(pattern, match_line, flags=re.IGNORECASE) if m: - for n in range(len(m.groups())+1): - replacement = re.sub(r'(\\{0:s}|\\g<{0:s}>)'.format(str(n)), - line[m.start(n):m.end(n)], - replacement) + for n in range(len(m.groups()) + 1): + replacement = re.sub( + r"(\\{0:s}|\\g<{0:s}>)".format(str(n)), + line[m.start(n) : m.end(n)], + replacement, + ) - line = "".join([line[:m.start(0)], - replacement, - line[m.end(0):]]) + line = "".join([line[: m.start(0)], replacement, line[m.end(0) :]]) workline = clean_str_continuation(line, str_continuation) @@ -153,8 +158,7 @@ def replace_comment_patterns(line, str_continuation): match_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:" \ - "".format(e.msg)) + print("{0:s} Line simplification has failed for:" "".format(e.msg)) print(line) exit(1) @@ -162,8 +166,8 @@ def replace_comment_patterns(line, str_continuation): def upcase_keywords(line, str_continuation): - '''Upper-case any Fortran keywords on the given line, and down-case any - all capital words which aren't keywords, returning the result''' + """Upper-case any Fortran keywords on the given line, and down-case any + all capital words which aren't keywords, returning the result""" workline = clean_str_continuation(line, str_continuation) @@ -195,20 +199,20 @@ def upcase_keywords(line, str_continuation): for word in line_words: # Exclude special "__FILE__" or "__LINE__" directives - if (word.isupper() and - not re.match(r"__\w+__", word)): - recomp = re.compile(r'(^|\b){0:s}(\b|$)'.format(word)) - simple_line = recomp.sub(r'\g<1>{0:s}' - r'\g<2>'.format(word.lower()), - simple_line) + if word.isupper() and not re.match(r"__\w+__", word): + recomp = re.compile(r"(^|\b){0:s}(\b|$)".format(word)) + simple_line = recomp.sub( + r"\g<1>{0:s}" r"\g<2>".format(word.lower()), simple_line + ) line_words = set([word.lower() for word in line_words]) words_to_upcase = list(line_words.intersection(KEYWORDS)) line = list(line) for keyword in words_to_upcase: - recomp = re.compile(r'(?i)(^|\b){0:s}(\b|$)'.format(keyword)) - simple_line = recomp.sub(r'\g<1>{0:s}\g<2>'.format(keyword.upper()), - simple_line) + recomp = re.compile(r"(?i)(^|\b){0:s}(\b|$)".format(keyword)) + simple_line = recomp.sub( + r"\g<1>{0:s}\g<2>".format(keyword.upper()), simple_line + ) # Now add back any comments/strings simple_line = list(simple_line) @@ -227,10 +231,10 @@ def upcase_keywords(line, str_continuation): def declaration_double_colon(iline, lines, pp_line_previous, line_previous): - ''' + """ Attempt to add the double colon to definition lines which do not already have it - ''' + """ line = lines[iline] @@ -240,8 +244,9 @@ def declaration_double_colon(iline, lines, pp_line_previous, line_previous): found_dec_type = None for declare_type in FORTRAN_TYPES: - if re.search(r"^\s*{0:s}\W".format(declare_type), - workline, flags=re.IGNORECASE): + if re.search( + r"^\s*{0:s}\W".format(declare_type), workline, flags=re.IGNORECASE + ): found_dec_type = declare_type break @@ -252,8 +257,7 @@ def declaration_double_colon(iline, lines, pp_line_previous, line_previous): # Pre-process the line to pull in any continuation lines simple_line = simplify_line(xlines) - if not re.search(r"\s+FUNCTION(,|\s|\()", - simple_line, flags=re.IGNORECASE): + if not re.search(r"\s+FUNCTION(,|\s|\()", simple_line, flags=re.IGNORECASE): # The presence of declaration attributes (ALLOCATABLE, # PUBLIC, POINTER, etc) are only valid when used with # the double-colon. Therefore after allowing for the @@ -261,8 +265,10 @@ def declaration_double_colon(iline, lines, pp_line_previous, line_previous): # older "*INT" declaration the first character should # not be a comma search = re.search( - r"^(\s*{0:s}\s*?(\(.*?\)|\*\s*[0-9]+|))\s+(\w+)".format( - found_dec_type), simple_line, flags=re.IGNORECASE) + r"^(\s*{0:s}\s*?(\(.*?\)|\*\s*[0-9]+|))\s+(\w+)".format(found_dec_type), + simple_line, + flags=re.IGNORECASE, + ) if search: # avoid CLASS IS, TYPE IS and CLASS DEFAULT statements classtype = search.group(3).strip().upper() @@ -279,24 +285,28 @@ def declaration_double_colon(iline, lines, pp_line_previous, line_previous): # Attempt to fit the double-colon into an existing space to # preserve indentation, otherwise just add it to the line - line = re.sub(r"^{0:s}" - r"\s\s\s\s".format(re.escape(statement)), - r"{0:s} :: ".format(statement), - line, count=1) - line = re.sub(r"^{0:s}\s*" - r"((?".format( - statement), line, count=1) + line = re.sub( + r"^{0:s}" r"\s\s\s\s".format(re.escape(statement)), + r"{0:s} :: ".format(statement), + line, + count=1, + ) + line = re.sub( + r"^{0:s}\s*" + r"((?".format(statement), + line, + count=1, + ) return line def apply_styling(lines): - ''' + """ For a lit of lines apply UMDP3 styling to each line and return the result - ''' + """ output_lines = [] continuation = False @@ -308,8 +318,7 @@ def apply_styling(lines): line_previous = "" for iline, line in enumerate(lines): - line = declaration_double_colon(iline, lines, pp_line_previous, - line_previous) + line = declaration_double_colon(iline, lines, pp_line_previous, line_previous) if pp_continuation: if not pseudo_comment: @@ -348,22 +357,22 @@ def apply_styling(lines): # if we are a (pp) continuation, save the partial line if pp_continuation: - pp_line_previous = ''.join([re.sub(r"\\\s*$", "", - pp_line_previous), - re.sub(r"&\s*$", "", line_previous), - line]) + pp_line_previous = "".join( + [ + re.sub(r"\\\s*$", "", pp_line_previous), + re.sub(r"&\s*$", "", line_previous), + line, + ] + ) line_previous = "" pseudo_line = re.sub(r"\\\s*$", "&", pp_line_previous) - pseudo_str_continuation = is_str_continuation(pseudo_line, - str_continuation) + pseudo_str_continuation = is_str_continuation(pseudo_line, str_continuation) if not pseudo_comment: - pseudo_line = partial_blank_fstring(pseudo_line, - str_continuation) + pseudo_line = partial_blank_fstring(pseudo_line, str_continuation) if pseudo_line.strip()[0] == "#": pseudo_comment = True if pseudo_line.find("!") != -1: - pseudo_line = blank_fcomments(pseudo_line, - str_continuation) + pseudo_line = blank_fcomments(pseudo_line, str_continuation) if pseudo_line.find("!") == -1: pseudo_comment = True elif continuation: @@ -376,10 +385,10 @@ def apply_styling(lines): def main(): - '''Main toplevel function for testing''' + """Main toplevel function for testing""" input_file = sys.argv[-1] with open(input_file, "r+") as file_in: - print("Styling "+input_file) + print("Styling " + input_file) lines_in = file_in.read().split("\n") new_lines = apply_styling(lines_in) file_in.seek(0) @@ -387,5 +396,5 @@ def main(): file_in.truncate() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/styling_keywords.py b/styling_keywords.py index 8cc7565d..82444565 100644 --- a/styling_keywords.py +++ b/styling_keywords.py @@ -9,222 +9,1426 @@ """ Newer keywords used during physics forking. """ -NEW_KEYWORDS = ['abort', 'abs', 'abstract', 'access', 'achar', 'acos', 'acosd', 'acosh', 'action', 'adjustl', - 'adjustr', 'advance', 'aimag', 'aint', 'alarm', 'algama', 'all', 'allocatable', 'allocate', - 'allocated', 'alog', 'alog10', 'amax0', 'amax1', 'amin0', 'amin1', 'amod', 'and', 'anint', 'any', - 'asin', 'asind', 'asinh', 'assign', 'assignment', 'associate', 'associated', 'asynchronous', 'atan', - 'atan2', 'atan2d', 'atand', 'atanh', 'atomic_add', 'atomic_and', 'atomic_cas', 'atomic_define', - 'atomic_fetch_add', 'atomic_fetch_and', 'atomic_fetch_or', 'atomic_fetch_xor', 'atomic_int_kind', - 'atomic_logical_kind', 'atomic_or', 'atomic_ref', 'atomic_xor', 'backspace', 'backtrace', 'besj0', - 'besj1', 'besjn', 'bessel_j0', 'bessel_j1', 'bessel_jn', 'bessel_y0', 'bessel_y1', 'bessel_yn', - 'besy0', 'besy1', 'besyn', 'bge', 'bgt', 'bind', 'bit_size', 'blank', 'ble', 'block', 'blt', - 'btest', 'cabs', 'call', 'case', 'ccos', 'cdabs', 'cdcos', 'cdexp', 'cdlog', 'cdsin', 'cdsqrt', - 'ceiling', 'cexp', 'char', 'character', 'character_kinds', 'character_storage_size', 'chdir', - 'chmod', 'class', 'clog', 'close', 'cmplx', 'codimension', 'command_argument_count', 'common', - 'compiler_options', 'compiler_version', 'complex', 'concurrent', 'conjg', 'contains', 'contiguous', - 'continue', 'convert', 'cos', 'cosd', 'cosh', 'cotan', 'cotand', 'count', 'co_broadcast', 'co_max', - 'co_min', 'co_reduce', 'co_sum', 'cpp', 'cpu_time', 'cqabs', 'cqcos', 'cqexp', 'cqlog', 'cqsin', - 'cqsqrt', 'cshift', 'csin', 'csqrt', 'ctime', 'cycle', 'c_alert', 'c_associated', 'c_backspace', - 'c_bool', 'c_carriage_return', 'c_char', 'c_double', 'c_double_complex', 'c_float', 'c_float128', - 'c_float128_complex', 'c_float_complex', 'c_form_feed', 'c_funloc', 'c_funptr', 'c_f_pointer', - 'c_f_procpointer', 'c_horizontal_tab', 'c_int', 'c_int128_t', 'c_int16_t', 'c_int32_t', 'c_int64_t', - 'c_int8_t', 'c_intmax_t', 'c_intptr_t', 'c_int_fast128_t', 'c_int_fast16_t', 'c_int_fast32_t', - 'c_int_fast64_t', 'c_int_fast8_t', 'c_int_least128_t', 'c_int_least16_t', 'c_int_least32_t', - 'c_int_least64_t', 'c_int_least8_t', 'c_loc', 'c_long', 'c_long_double', 'c_long_double_complex', - 'c_long_long', 'c_new_line', 'c_null_char', 'c_null_funptr', 'c_null_ptr', 'c_ptr', 'c_ptrdiff_t', - 'c_short', 'c_signed_char', 'c_sizeof', 'c_size_t', 'c_vertical_tab', 'dabs', 'dacos', 'dacosh', - 'dasin', 'dasinh', 'data', 'datan', 'datan2', 'datanh', 'date_and_time', 'dbesj0', 'dbesj1', - 'dbesjn', 'dbesy0', 'dbesy1', 'dbesyn', 'dble', 'dcmplx', 'dconjg', 'dcos', 'dcosh', 'ddim', - 'deallocate', 'decode', 'deferred', 'delim', 'derf', 'derfc', 'dexp', 'dfloat', 'dgamma', 'digits', - 'dim', 'dimag', 'dimension', 'dint', 'direct', 'dlgama', 'dlog', 'dlog10', 'dmax1', 'dmin1', 'dmod', - 'dnint', 'do', 'dot_product', 'double', 'dprod', 'dreal', 'dshiftl', 'dshiftr', 'dsign', 'dsin', - 'dsinh', 'dsqrt', 'dtan', 'dtanh', 'dtime', 'elemental', 'else', 'encode', 'end', 'entry', 'enum', - 'enumerator', 'eor', 'eoshift', 'epsilon', 'eq', 'equivalence', 'eqv', 'erf', 'erfc', 'erfc_scaled', - 'errmsg', 'error', 'error_unit', 'etime', 'event_query', 'execute_command_line', 'exist', 'exit', - 'exp', 'exponent', 'extends', 'extends_type_of', 'external', 'false', 'fdate', 'fget', 'fgetc', - 'file', 'file_storage_size', 'final', 'float', 'floor', 'flush', 'fmt', 'fnum', 'forall', 'form', - 'format', 'formatted', 'fpp', 'fput', 'fputc', 'fraction', 'free', 'fseek', 'fstat', 'ftell', - 'function', 'gamma', 'ge', 'generic', 'gerror', 'getarg', 'getcwd', 'getenv', 'getgid', 'getlog', - 'getpid', 'getuid', 'get_command', 'get_command_argument', 'get_environment_variable', 'gmtime', - 'go', 'gt', 'hostnm', 'huge', 'hypot', 'iabs', 'iachar', 'iall', 'iand', 'iany', 'iargc', 'ibclr', - 'ibits', 'ibset', 'ichar', 'idate', 'idim', 'idint', 'idnint', 'ieee_class', 'ieee_class_type', - 'ieee_copy_sign', 'ieee_is_finite', 'ieee_is_nan', 'ieee_is_negative', 'ieee_is_normal', - 'ieee_logb', 'ieee_negative_denormal', 'ieee_negative_inf', 'ieee_negative_normal', - 'ieee_negative_zero', 'ieee_next_after', 'ieee_positive_denormal', 'ieee_positive_inf', - 'ieee_positive_normal', 'ieee_positive_zero', 'ieee_quiet_nan', 'ieee_rem', 'ieee_rint', - 'ieee_scalb', 'ieee_selected_real_kind', 'ieee_signaling_nan', 'ieee_support_datatype', - 'ieee_support_denormal', 'ieee_support_divide', 'ieee_support_inf', 'ieee_support_nan', - 'ieee_support_sqrt', 'ieee_support_standard', 'ieee_unordered', 'ieee_value', 'ieor', 'ierrno', - 'if', 'ifix', 'imag', 'images', 'image_index', 'imagpart', 'implicit', 'import', 'in', 'include', - 'index', 'input_unit', 'inquire', 'int', 'int16', 'int2', 'int32', 'int64', 'int8', 'integer', - 'integer_kinds', 'intent', 'interface', 'intrinsic', 'iomsg', 'ior', 'iostat', 'iostat_end', - 'iostat_eor', 'iostat_inquire_internal_unit', 'iparity', 'iqint', 'irand', 'is', 'isatty', 'ishft', - 'ishftc', 'isign', 'isnan', 'iso_c_binding', 'iso_fortran_env', 'is_iostat_end', 'is_iostat_eor', - 'itime', 'kill', 'kind', 'lbound', 'lcobound', 'le', 'leadz', 'len', 'len_trim', 'lgamma', 'lge', - 'lgt', 'link', 'lle', 'llt', 'lnblnk', 'loc', 'lock', 'lock_type', 'log', 'log10', 'logical', - 'logical_kinds', 'log_gamma', 'long', 'lshift', 'lstat', 'lt', 'ltime', 'malloc', 'maskl', 'maskr', - 'matmul', 'max', 'max0', 'max1', 'maxexponent', 'maxloc', 'maxval', 'mclock', 'mclock8', 'memory', - 'merge', 'merge_bits', 'min', 'min0', 'min1', 'minexponent', 'minloc', 'minval', 'mod', 'module', - 'modulo', 'move_alloc', 'mvbits', 'name', 'named', 'namelist', 'ne', 'nearest', 'neqv', 'new_line', - 'nextrec', 'nint', 'nml', 'none', 'non_intrinsic', 'non_overridable', 'nopass', 'norm2', 'not', - 'null', 'nullify', 'number', 'numeric_storage_size', 'num_images', 'only', 'open', 'opened', - 'operator', 'optional', 'or', 'out', 'output_unit', 'pack', 'pad', 'parameter', 'parity', 'pass', - 'perror', 'pointer', 'popcnt', 'poppar', 'position', 'precision', 'present', 'print', 'private', - 'procedure', 'product', 'program', 'protected', 'public', 'pure', 'qabs', 'qacos', 'qasin', 'qatan', - 'qatan2', 'qcmplx', 'qconjg', 'qcos', 'qcosh', 'qdim', 'qerf', 'qerfc', 'qexp', 'qgamma', 'qimag', - 'qlgama', 'qlog', 'qlog10', 'qmax1', 'qmin1', 'qmod', 'qnint', 'qsign', 'qsin', 'qsinh', 'qsqrt', - 'qtan', 'qtanh', 'radix', 'ran', 'rand', 'random_number', 'random_seed', 'range', 'rank', 'read', - 'readwrite', 'real', 'real128', 'real32', 'real64', 'realpart', 'real_kinds', 'rec', 'recl', - 'record', 'recursive', 'rename', 'repeat', 'reshape', 'result', 'return', 'rewind', 'rewrite', - 'rrspacing', 'rshift', 'same_type_as', 'save', 'scale', 'scan', 'secnds', 'second', 'select', - 'selected_char_kind', 'selected_int_kind', 'selected_real_kind', 'sequence', 'sequential', - 'set_exponent', 'shape', 'shifta', 'shiftl', 'shiftr', 'short', 'sign', 'signal', 'sin', 'sind', - 'sinh', 'size', 'sizeof', 'sleep', 'sngl', 'source', 'spacing', 'spread', 'sqrt', 'srand', 'stat', - 'status', 'stat_failed_image', 'stat_locked', 'stat_locked_other_image', 'stat_stopped_image', - 'stat_unlocked', 'stop', 'storage_size', 'structure', 'submodule', 'subroutine', 'sum', 'symlnk', - 'sync', 'system', 'system_clock', 'tan', 'tand', 'tanh', 'target', 'then', 'this_image', 'time', - 'time8', 'tiny', 'to', 'trailz', 'transfer', 'transpose', 'trim', 'true', 'ttynam', 'type', - 'ubound', 'ucobound', 'umask', 'unformatted', 'unit', 'unlink', 'unlock', 'unpack', 'use', 'value', - 'verif', 'verify', 'volatile', 'wait', 'where', 'while', 'write', 'xor', 'zabs', 'zcos', 'zexp', - 'zlog', 'zsin', 'zsqrt', '.and.', '.eqv.', '.eq.', '.false.', '.ge.', '.gt.', '.le.', '.lt.', - '.neqv.', '.ne.', '.not.', '.or.', '.true.', '.xor.'] +NEW_KEYWORDS = [ + "abort", + "abs", + "abstract", + "access", + "achar", + "acos", + "acosd", + "acosh", + "action", + "adjustl", + "adjustr", + "advance", + "aimag", + "aint", + "alarm", + "algama", + "all", + "allocatable", + "allocate", + "allocated", + "alog", + "alog10", + "amax0", + "amax1", + "amin0", + "amin1", + "amod", + "and", + "anint", + "any", + "asin", + "asind", + "asinh", + "assign", + "assignment", + "associate", + "associated", + "asynchronous", + "atan", + "atan2", + "atan2d", + "atand", + "atanh", + "atomic_add", + "atomic_and", + "atomic_cas", + "atomic_define", + "atomic_fetch_add", + "atomic_fetch_and", + "atomic_fetch_or", + "atomic_fetch_xor", + "atomic_int_kind", + "atomic_logical_kind", + "atomic_or", + "atomic_ref", + "atomic_xor", + "backspace", + "backtrace", + "besj0", + "besj1", + "besjn", + "bessel_j0", + "bessel_j1", + "bessel_jn", + "bessel_y0", + "bessel_y1", + "bessel_yn", + "besy0", + "besy1", + "besyn", + "bge", + "bgt", + "bind", + "bit_size", + "blank", + "ble", + "block", + "blt", + "btest", + "cabs", + "call", + "case", + "ccos", + "cdabs", + "cdcos", + "cdexp", + "cdlog", + "cdsin", + "cdsqrt", + "ceiling", + "cexp", + "char", + "character", + "character_kinds", + "character_storage_size", + "chdir", + "chmod", + "class", + "clog", + "close", + "cmplx", + "codimension", + "command_argument_count", + "common", + "compiler_options", + "compiler_version", + "complex", + "concurrent", + "conjg", + "contains", + "contiguous", + "continue", + "convert", + "cos", + "cosd", + "cosh", + "cotan", + "cotand", + "count", + "co_broadcast", + "co_max", + "co_min", + "co_reduce", + "co_sum", + "cpp", + "cpu_time", + "cqabs", + "cqcos", + "cqexp", + "cqlog", + "cqsin", + "cqsqrt", + "cshift", + "csin", + "csqrt", + "ctime", + "cycle", + "c_alert", + "c_associated", + "c_backspace", + "c_bool", + "c_carriage_return", + "c_char", + "c_double", + "c_double_complex", + "c_float", + "c_float128", + "c_float128_complex", + "c_float_complex", + "c_form_feed", + "c_funloc", + "c_funptr", + "c_f_pointer", + "c_f_procpointer", + "c_horizontal_tab", + "c_int", + "c_int128_t", + "c_int16_t", + "c_int32_t", + "c_int64_t", + "c_int8_t", + "c_intmax_t", + "c_intptr_t", + "c_int_fast128_t", + "c_int_fast16_t", + "c_int_fast32_t", + "c_int_fast64_t", + "c_int_fast8_t", + "c_int_least128_t", + "c_int_least16_t", + "c_int_least32_t", + "c_int_least64_t", + "c_int_least8_t", + "c_loc", + "c_long", + "c_long_double", + "c_long_double_complex", + "c_long_long", + "c_new_line", + "c_null_char", + "c_null_funptr", + "c_null_ptr", + "c_ptr", + "c_ptrdiff_t", + "c_short", + "c_signed_char", + "c_sizeof", + "c_size_t", + "c_vertical_tab", + "dabs", + "dacos", + "dacosh", + "dasin", + "dasinh", + "data", + "datan", + "datan2", + "datanh", + "date_and_time", + "dbesj0", + "dbesj1", + "dbesjn", + "dbesy0", + "dbesy1", + "dbesyn", + "dble", + "dcmplx", + "dconjg", + "dcos", + "dcosh", + "ddim", + "deallocate", + "decode", + "deferred", + "delim", + "derf", + "derfc", + "dexp", + "dfloat", + "dgamma", + "digits", + "dim", + "dimag", + "dimension", + "dint", + "direct", + "dlgama", + "dlog", + "dlog10", + "dmax1", + "dmin1", + "dmod", + "dnint", + "do", + "dot_product", + "double", + "dprod", + "dreal", + "dshiftl", + "dshiftr", + "dsign", + "dsin", + "dsinh", + "dsqrt", + "dtan", + "dtanh", + "dtime", + "elemental", + "else", + "encode", + "end", + "entry", + "enum", + "enumerator", + "eor", + "eoshift", + "epsilon", + "eq", + "equivalence", + "eqv", + "erf", + "erfc", + "erfc_scaled", + "errmsg", + "error", + "error_unit", + "etime", + "event_query", + "execute_command_line", + "exist", + "exit", + "exp", + "exponent", + "extends", + "extends_type_of", + "external", + "false", + "fdate", + "fget", + "fgetc", + "file", + "file_storage_size", + "final", + "float", + "floor", + "flush", + "fmt", + "fnum", + "forall", + "form", + "format", + "formatted", + "fpp", + "fput", + "fputc", + "fraction", + "free", + "fseek", + "fstat", + "ftell", + "function", + "gamma", + "ge", + "generic", + "gerror", + "getarg", + "getcwd", + "getenv", + "getgid", + "getlog", + "getpid", + "getuid", + "get_command", + "get_command_argument", + "get_environment_variable", + "gmtime", + "go", + "gt", + "hostnm", + "huge", + "hypot", + "iabs", + "iachar", + "iall", + "iand", + "iany", + "iargc", + "ibclr", + "ibits", + "ibset", + "ichar", + "idate", + "idim", + "idint", + "idnint", + "ieee_class", + "ieee_class_type", + "ieee_copy_sign", + "ieee_is_finite", + "ieee_is_nan", + "ieee_is_negative", + "ieee_is_normal", + "ieee_logb", + "ieee_negative_denormal", + "ieee_negative_inf", + "ieee_negative_normal", + "ieee_negative_zero", + "ieee_next_after", + "ieee_positive_denormal", + "ieee_positive_inf", + "ieee_positive_normal", + "ieee_positive_zero", + "ieee_quiet_nan", + "ieee_rem", + "ieee_rint", + "ieee_scalb", + "ieee_selected_real_kind", + "ieee_signaling_nan", + "ieee_support_datatype", + "ieee_support_denormal", + "ieee_support_divide", + "ieee_support_inf", + "ieee_support_nan", + "ieee_support_sqrt", + "ieee_support_standard", + "ieee_unordered", + "ieee_value", + "ieor", + "ierrno", + "if", + "ifix", + "imag", + "images", + "image_index", + "imagpart", + "implicit", + "import", + "in", + "include", + "index", + "input_unit", + "inquire", + "int", + "int16", + "int2", + "int32", + "int64", + "int8", + "integer", + "integer_kinds", + "intent", + "interface", + "intrinsic", + "iomsg", + "ior", + "iostat", + "iostat_end", + "iostat_eor", + "iostat_inquire_internal_unit", + "iparity", + "iqint", + "irand", + "is", + "isatty", + "ishft", + "ishftc", + "isign", + "isnan", + "iso_c_binding", + "iso_fortran_env", + "is_iostat_end", + "is_iostat_eor", + "itime", + "kill", + "kind", + "lbound", + "lcobound", + "le", + "leadz", + "len", + "len_trim", + "lgamma", + "lge", + "lgt", + "link", + "lle", + "llt", + "lnblnk", + "loc", + "lock", + "lock_type", + "log", + "log10", + "logical", + "logical_kinds", + "log_gamma", + "long", + "lshift", + "lstat", + "lt", + "ltime", + "malloc", + "maskl", + "maskr", + "matmul", + "max", + "max0", + "max1", + "maxexponent", + "maxloc", + "maxval", + "mclock", + "mclock8", + "memory", + "merge", + "merge_bits", + "min", + "min0", + "min1", + "minexponent", + "minloc", + "minval", + "mod", + "module", + "modulo", + "move_alloc", + "mvbits", + "name", + "named", + "namelist", + "ne", + "nearest", + "neqv", + "new_line", + "nextrec", + "nint", + "nml", + "none", + "non_intrinsic", + "non_overridable", + "nopass", + "norm2", + "not", + "null", + "nullify", + "number", + "numeric_storage_size", + "num_images", + "only", + "open", + "opened", + "operator", + "optional", + "or", + "out", + "output_unit", + "pack", + "pad", + "parameter", + "parity", + "pass", + "perror", + "pointer", + "popcnt", + "poppar", + "position", + "precision", + "present", + "print", + "private", + "procedure", + "product", + "program", + "protected", + "public", + "pure", + "qabs", + "qacos", + "qasin", + "qatan", + "qatan2", + "qcmplx", + "qconjg", + "qcos", + "qcosh", + "qdim", + "qerf", + "qerfc", + "qexp", + "qgamma", + "qimag", + "qlgama", + "qlog", + "qlog10", + "qmax1", + "qmin1", + "qmod", + "qnint", + "qsign", + "qsin", + "qsinh", + "qsqrt", + "qtan", + "qtanh", + "radix", + "ran", + "rand", + "random_number", + "random_seed", + "range", + "rank", + "read", + "readwrite", + "real", + "real128", + "real32", + "real64", + "realpart", + "real_kinds", + "rec", + "recl", + "record", + "recursive", + "rename", + "repeat", + "reshape", + "result", + "return", + "rewind", + "rewrite", + "rrspacing", + "rshift", + "same_type_as", + "save", + "scale", + "scan", + "secnds", + "second", + "select", + "selected_char_kind", + "selected_int_kind", + "selected_real_kind", + "sequence", + "sequential", + "set_exponent", + "shape", + "shifta", + "shiftl", + "shiftr", + "short", + "sign", + "signal", + "sin", + "sind", + "sinh", + "size", + "sizeof", + "sleep", + "sngl", + "source", + "spacing", + "spread", + "sqrt", + "srand", + "stat", + "status", + "stat_failed_image", + "stat_locked", + "stat_locked_other_image", + "stat_stopped_image", + "stat_unlocked", + "stop", + "storage_size", + "structure", + "submodule", + "subroutine", + "sum", + "symlnk", + "sync", + "system", + "system_clock", + "tan", + "tand", + "tanh", + "target", + "then", + "this_image", + "time", + "time8", + "tiny", + "to", + "trailz", + "transfer", + "transpose", + "trim", + "true", + "ttynam", + "type", + "ubound", + "ucobound", + "umask", + "unformatted", + "unit", + "unlink", + "unlock", + "unpack", + "use", + "value", + "verif", + "verify", + "volatile", + "wait", + "where", + "while", + "write", + "xor", + "zabs", + "zcos", + "zexp", + "zlog", + "zsin", + "zsqrt", + ".and.", + ".eqv.", + ".eq.", + ".false.", + ".ge.", + ".gt.", + ".le.", + ".lt.", + ".neqv.", + ".ne.", + ".not.", + ".or.", + ".true.", + ".xor.", +] -KEYWORDS = {"abort", "abs", "abstract", "access", "achar", "acos", "acosd", "acosh", "action", "adjustl", "adjustr", - "advance", "aimag", "aint", "alarm", "algama", "all", "allocatable", "allocate", "allocated", "alog", - "alog10", "amax0", "amax1", "amin0", "amin1", "amod", "and", "anint", "any", "asin", "asind", "asinh", - "assign", "assignment", "associate", "associated", "asynchronous", "atan", "atan2", "atan2d", "atand", - "atanh", "atomic", "atomic_add", "atomic_and", "atomic_cas", "atomic_define", "atomic_fetch_add", - "atomic_fetch_and", "atomic_fetch_or", "atomic_fetch_xor", "atomic_int_kind", "atomic_logical_kind", - "atomic_or", "atomic_ref", "atomic_xor", "backspace", "backtrace", "barrier", "besj0", "besj1", "besjn", - "bessel_j0", "bessel_j1", "bessel_jn", "bessel_y0", "bessel_y1", "bessel_yn", "besy0", "besy1", "besyn", - "bge", "bgt", "bind", "bit_size", "blank", "ble", "block", "blt", "btest", "c_alert", "c_associated", - "c_backspace", "c_bool", "c_carriage_return", "c_char", "c_double", "c_double_complex", "c_f_pointer", - "c_f_procpointer", "c_float", "c_float128", "c_float128_complex", "c_float_complex", "c_form_feed", - "c_funloc", "c_funptr", "c_horizontal_tab", "c_int", "c_int128_t", "c_int16_t", "c_int32_t", "c_int64_t", - "c_int8_t", "c_int_fast128_t", "c_int_fast16_t", "c_int_fast32_t", "c_int_fast64_t", "c_int_fast8_t", - "c_int_least128_t", "c_int_least16_t", "c_int_least32_t", "c_int_least64_t", "c_int_least8_t", "c_intmax_t", - "c_intptr_t", "c_loc", "c_long", "c_long_double", "c_long_double_complex", "c_long_long", "c_new_line", - "c_null_char", "c_null_funptr", "c_null_ptr", "c_ptr", "c_ptrdiff_t", "c_short", "c_signed_char", - "c_size_t", "c_sizeof", "c_vertical_tab", "cabs", "call", "case", "ccos", "cdabs", "cdcos", "cdexp", - "cdlog", "cdsin", "cdsqrt", "ceiling", "cexp", "char", "character", "character_kinds", - "character_storage_size", "chdir", "chmod", "class", "clog", "close", "cmplx", "co_broadcast", "co_max", - "co_min", "co_reduce", "co_sum", "codimension", "command_argument_count", "common", "compiler_options", - "compiler_version", "complex", "concurrent", "conjg", "contains", "contiguous", "continue", "convert", - "copyin", "copyprivate", "cos", "cosd", "cosh", "cotan", "cotand", "count", "cpp", "cpu_time", "cqabs", - "cqcos", "cqexp", "cqlog", "cqsin", "cqsqrt", "critical", "cshift", "csin", "csqrt", "ctime", "cycle", - "dabs", "dacos", "dacosh", "dasin", "dasinh", "data", "datan", "datan2", "datanh", "date_and_time", - "dbesj0", "dbesj1", "dbesjn", "dbesy0", "dbesy1", "dbesyn", "dble", "dcmplx", "dconjg", "dcos", "dcosh", - "ddim", "deallocate", "decode", "default", "deferred", "delim", "derf", "derfc", "dexp", "dfloat", "dgamma", - "digits", "dim", "dimag", "dimension", "dint", "direct", "dlgama", "dlog", "dlog10", "dmax1", "dmin1", - "dmod", "dnint", "do", "dot_product", "double", "dprod", "dreal", "dshiftl", "dshiftr", "dsign", "dsin", - "dsinh", "dsqrt", "dtan", "dtanh", "dtime", "elemental", "else", "elsewhere", "encode", "end", "endfile", - "entry", "enum", "enumerator", "eor", "eoshift", "epsilon", "equivalence", "eqv", "erf", "erfc", - "erfc_scaled", "errmsg", "error", "error_unit", "etime", "event_query", "execute_command_line", "exist", - "exit", "exp", "exponent", "extends", "extends_type_of", "external", "false", "fdate", "fget", "fgetc", - "file", "file_storage_size", "final", "firstprivate", "float", "floor", "flush", "fmt", "fnum", "forall", - "form", "format", "formatted", "fpp", "fput", "fputc", "fraction", "free", "fseek", "fstat", "ftell", - "function", "gamma", "generic", "gerror", "get_command", "get_command_argument", "get_environment_variable", - "getarg", "getcwd", "getenv", "getgid", "getlog", "getpid", "getuid", "gmtime", "go", "hostnm", "huge", - "hypot", "iabs", "iachar", "iall", "iand", "iany", "iargc", "ibclr", "ibits", "ibset", "ichar", "idate", - "idim", "idint", "idnint", "ieee_class", "ieee_class_type", "ieee_copy_sign", "ieee_is_finite", - "ieee_is_nan", "ieee_is_negative", "ieee_is_normal", "ieee_logb", "ieee_negative_denormal", - "ieee_negative_inf", "ieee_negative_normal", "ieee_negative_zero", "ieee_next_after", - "ieee_positive_denormal", "ieee_positive_inf", "ieee_positive_normal", "ieee_positive_zero", - "ieee_quiet_nan", "ieee_rem", "ieee_rint", "ieee_scalb", "ieee_selected_real_kind", "ieee_signaling_nan", - "ieee_support_datatype", "ieee_support_denormal", "ieee_support_divide", "ieee_support_inf", - "ieee_support_nan", "ieee_support_sqrt", "ieee_support_standard", "ieee_unordered", "ieee_value", "ieor", - "ierrno", "if", "ifix", "imag", "image_index", "images", "imagpart", "implicit", "import", "in", "include", - "index", "inout", "input_unit", "inquire", "int", "int16", "int2", "int32", "int64", "int8", "integer", - "integer_kinds", "intent", "interface", "intrinsic", "iomsg", "ior", "iostat", "iostat_end", "iostat_eor", - "iostat_inquire_internal_unit", "iparity", "iqint", "irand", "is", "is_iostat_end", "is_iostat_eor", - "isatty", "ishft", "ishftc", "isign", "isnan", "iso_c_binding", "iso_fortran_env", "itime", "kill", "kind", - "lastprivate", "lbound", "lcobound", "leadz", "len", "len_trim", "lgamma", "lge", "lgt", "link", "lle", - "llt", "lnblnk", "loc", "lock", "lock_type", "log", "log10", "log_gamma", "logical", "logical_kinds", - "long", "lshift", "lstat", "ltime", "malloc", "maskl", "maskr", "master", "matmul", "max", "max0", "max1", - "maxexponent", "maxloc", "maxval", "mclock", "mclock8", "memory", "merge", "merge_bits", "min", "min0", - "min1", "minexponent", "minloc", "minval", "mod", "module", "modulo", "move_alloc", "mvbits", "name", - "named", "namelist", "nearest", "neqv", "new_line", "nextrec", "nint", "nml", "non_intrinsic", - "non_overridable", "none", "nopass", "norm2", "not", "null", "nullify", "num_images", "number", - "numeric_storage_size", "only", "open", "opened", "operator", "optional", "or", "ordered", "out", - "output_unit", "pack", "pad", "parallel", "parameter", "parity", "pass", "perror", "pointer", "popcnt", - "poppar", "position", "precision", "present", "print", "private", "procedure", "product", "program", - "protected", "public", "pure", "qabs", "qacos", "qasin", "qatan", "qatan2", "qcmplx", "qconjg", "qcos", - "qcosh", "qdim", "qerf", "qerfc", "qexp", "qgamma", "qimag", "qlgama", "qlog", "qlog10", "qmax1", "qmin1", - "qmod", "qnint", "qsign", "qsin", "qsinh", "qsqrt", "qtan", "qtanh", "radix", "ran", "rand", - "random_number", "random_seed", "range", "rank", "read", "readwrite", "real", "real128", "real32", "real64", - "real_kinds", "realpart", "rec", "recl", "record", "recursive", "reduction", "rename", "repeat", "reshape", - "result", "return", "rewind", "rewrite", "rrspacing", "rshift", "same_type_as", "save", "scale", "scan", - "secnds", "second", "sections", "select", "selected_char_kind", "selected_int_kind", "selected_real_kind", - "sequence", "sequential", "set_exponent", "shape", "shared", "shifta", "shiftl", "shiftr", "short", "sign", - "signal", "sin", "sind", "sinh", "size", "sizeof", "sleep", "sngl", "source", "spacing", "spread", "sqrt", - "srand", "stat", "stat_failed_image", "stat_locked", "stat_locked_other_image", "stat_stopped_image", - "stat_unlocked", "status", "stop", "storage_size", "structure", "submodule", "subroutine", "sum", "symlnk", - "sync", "system", "system_clock", "tan", "tand", "tanh", "target", "task", "taskwait", "then", "this_image", - "threadprivate", "time", "time8", "tiny", "to", "trailz", "transfer", "transpose", "trim", "true", "ttynam", - "type", "ubound", "ucobound", "umask", "unformatted", "unit", "unlink", "unlock", "unpack", "use", "value", - "verif", "verify", "volatile", "wait", "where", "while", "workshare", "write", "xor", "zabs", "zcos", - "zexp", "zlog", "zsin", "zsqrt"} +KEYWORDS = { + "abort", + "abs", + "abstract", + "access", + "achar", + "acos", + "acosd", + "acosh", + "action", + "adjustl", + "adjustr", + "advance", + "aimag", + "aint", + "alarm", + "algama", + "all", + "allocatable", + "allocate", + "allocated", + "alog", + "alog10", + "amax0", + "amax1", + "amin0", + "amin1", + "amod", + "and", + "anint", + "any", + "asin", + "asind", + "asinh", + "assign", + "assignment", + "associate", + "associated", + "asynchronous", + "atan", + "atan2", + "atan2d", + "atand", + "atanh", + "atomic", + "atomic_add", + "atomic_and", + "atomic_cas", + "atomic_define", + "atomic_fetch_add", + "atomic_fetch_and", + "atomic_fetch_or", + "atomic_fetch_xor", + "atomic_int_kind", + "atomic_logical_kind", + "atomic_or", + "atomic_ref", + "atomic_xor", + "backspace", + "backtrace", + "barrier", + "besj0", + "besj1", + "besjn", + "bessel_j0", + "bessel_j1", + "bessel_jn", + "bessel_y0", + "bessel_y1", + "bessel_yn", + "besy0", + "besy1", + "besyn", + "bge", + "bgt", + "bind", + "bit_size", + "blank", + "ble", + "block", + "blt", + "btest", + "c_alert", + "c_associated", + "c_backspace", + "c_bool", + "c_carriage_return", + "c_char", + "c_double", + "c_double_complex", + "c_f_pointer", + "c_f_procpointer", + "c_float", + "c_float128", + "c_float128_complex", + "c_float_complex", + "c_form_feed", + "c_funloc", + "c_funptr", + "c_horizontal_tab", + "c_int", + "c_int128_t", + "c_int16_t", + "c_int32_t", + "c_int64_t", + "c_int8_t", + "c_int_fast128_t", + "c_int_fast16_t", + "c_int_fast32_t", + "c_int_fast64_t", + "c_int_fast8_t", + "c_int_least128_t", + "c_int_least16_t", + "c_int_least32_t", + "c_int_least64_t", + "c_int_least8_t", + "c_intmax_t", + "c_intptr_t", + "c_loc", + "c_long", + "c_long_double", + "c_long_double_complex", + "c_long_long", + "c_new_line", + "c_null_char", + "c_null_funptr", + "c_null_ptr", + "c_ptr", + "c_ptrdiff_t", + "c_short", + "c_signed_char", + "c_size_t", + "c_sizeof", + "c_vertical_tab", + "cabs", + "call", + "case", + "ccos", + "cdabs", + "cdcos", + "cdexp", + "cdlog", + "cdsin", + "cdsqrt", + "ceiling", + "cexp", + "char", + "character", + "character_kinds", + "character_storage_size", + "chdir", + "chmod", + "class", + "clog", + "close", + "cmplx", + "co_broadcast", + "co_max", + "co_min", + "co_reduce", + "co_sum", + "codimension", + "command_argument_count", + "common", + "compiler_options", + "compiler_version", + "complex", + "concurrent", + "conjg", + "contains", + "contiguous", + "continue", + "convert", + "copyin", + "copyprivate", + "cos", + "cosd", + "cosh", + "cotan", + "cotand", + "count", + "cpp", + "cpu_time", + "cqabs", + "cqcos", + "cqexp", + "cqlog", + "cqsin", + "cqsqrt", + "critical", + "cshift", + "csin", + "csqrt", + "ctime", + "cycle", + "dabs", + "dacos", + "dacosh", + "dasin", + "dasinh", + "data", + "datan", + "datan2", + "datanh", + "date_and_time", + "dbesj0", + "dbesj1", + "dbesjn", + "dbesy0", + "dbesy1", + "dbesyn", + "dble", + "dcmplx", + "dconjg", + "dcos", + "dcosh", + "ddim", + "deallocate", + "decode", + "default", + "deferred", + "delim", + "derf", + "derfc", + "dexp", + "dfloat", + "dgamma", + "digits", + "dim", + "dimag", + "dimension", + "dint", + "direct", + "dlgama", + "dlog", + "dlog10", + "dmax1", + "dmin1", + "dmod", + "dnint", + "do", + "dot_product", + "double", + "dprod", + "dreal", + "dshiftl", + "dshiftr", + "dsign", + "dsin", + "dsinh", + "dsqrt", + "dtan", + "dtanh", + "dtime", + "elemental", + "else", + "elsewhere", + "encode", + "end", + "endfile", + "entry", + "enum", + "enumerator", + "eor", + "eoshift", + "epsilon", + "equivalence", + "eqv", + "erf", + "erfc", + "erfc_scaled", + "errmsg", + "error", + "error_unit", + "etime", + "event_query", + "execute_command_line", + "exist", + "exit", + "exp", + "exponent", + "extends", + "extends_type_of", + "external", + "false", + "fdate", + "fget", + "fgetc", + "file", + "file_storage_size", + "final", + "firstprivate", + "float", + "floor", + "flush", + "fmt", + "fnum", + "forall", + "form", + "format", + "formatted", + "fpp", + "fput", + "fputc", + "fraction", + "free", + "fseek", + "fstat", + "ftell", + "function", + "gamma", + "generic", + "gerror", + "get_command", + "get_command_argument", + "get_environment_variable", + "getarg", + "getcwd", + "getenv", + "getgid", + "getlog", + "getpid", + "getuid", + "gmtime", + "go", + "hostnm", + "huge", + "hypot", + "iabs", + "iachar", + "iall", + "iand", + "iany", + "iargc", + "ibclr", + "ibits", + "ibset", + "ichar", + "idate", + "idim", + "idint", + "idnint", + "ieee_class", + "ieee_class_type", + "ieee_copy_sign", + "ieee_is_finite", + "ieee_is_nan", + "ieee_is_negative", + "ieee_is_normal", + "ieee_logb", + "ieee_negative_denormal", + "ieee_negative_inf", + "ieee_negative_normal", + "ieee_negative_zero", + "ieee_next_after", + "ieee_positive_denormal", + "ieee_positive_inf", + "ieee_positive_normal", + "ieee_positive_zero", + "ieee_quiet_nan", + "ieee_rem", + "ieee_rint", + "ieee_scalb", + "ieee_selected_real_kind", + "ieee_signaling_nan", + "ieee_support_datatype", + "ieee_support_denormal", + "ieee_support_divide", + "ieee_support_inf", + "ieee_support_nan", + "ieee_support_sqrt", + "ieee_support_standard", + "ieee_unordered", + "ieee_value", + "ieor", + "ierrno", + "if", + "ifix", + "imag", + "image_index", + "images", + "imagpart", + "implicit", + "import", + "in", + "include", + "index", + "inout", + "input_unit", + "inquire", + "int", + "int16", + "int2", + "int32", + "int64", + "int8", + "integer", + "integer_kinds", + "intent", + "interface", + "intrinsic", + "iomsg", + "ior", + "iostat", + "iostat_end", + "iostat_eor", + "iostat_inquire_internal_unit", + "iparity", + "iqint", + "irand", + "is", + "is_iostat_end", + "is_iostat_eor", + "isatty", + "ishft", + "ishftc", + "isign", + "isnan", + "iso_c_binding", + "iso_fortran_env", + "itime", + "kill", + "kind", + "lastprivate", + "lbound", + "lcobound", + "leadz", + "len", + "len_trim", + "lgamma", + "lge", + "lgt", + "link", + "lle", + "llt", + "lnblnk", + "loc", + "lock", + "lock_type", + "log", + "log10", + "log_gamma", + "logical", + "logical_kinds", + "long", + "lshift", + "lstat", + "ltime", + "malloc", + "maskl", + "maskr", + "master", + "matmul", + "max", + "max0", + "max1", + "maxexponent", + "maxloc", + "maxval", + "mclock", + "mclock8", + "memory", + "merge", + "merge_bits", + "min", + "min0", + "min1", + "minexponent", + "minloc", + "minval", + "mod", + "module", + "modulo", + "move_alloc", + "mvbits", + "name", + "named", + "namelist", + "nearest", + "neqv", + "new_line", + "nextrec", + "nint", + "nml", + "non_intrinsic", + "non_overridable", + "none", + "nopass", + "norm2", + "not", + "null", + "nullify", + "num_images", + "number", + "numeric_storage_size", + "only", + "open", + "opened", + "operator", + "optional", + "or", + "ordered", + "out", + "output_unit", + "pack", + "pad", + "parallel", + "parameter", + "parity", + "pass", + "perror", + "pointer", + "popcnt", + "poppar", + "position", + "precision", + "present", + "print", + "private", + "procedure", + "product", + "program", + "protected", + "public", + "pure", + "qabs", + "qacos", + "qasin", + "qatan", + "qatan2", + "qcmplx", + "qconjg", + "qcos", + "qcosh", + "qdim", + "qerf", + "qerfc", + "qexp", + "qgamma", + "qimag", + "qlgama", + "qlog", + "qlog10", + "qmax1", + "qmin1", + "qmod", + "qnint", + "qsign", + "qsin", + "qsinh", + "qsqrt", + "qtan", + "qtanh", + "radix", + "ran", + "rand", + "random_number", + "random_seed", + "range", + "rank", + "read", + "readwrite", + "real", + "real128", + "real32", + "real64", + "real_kinds", + "realpart", + "rec", + "recl", + "record", + "recursive", + "reduction", + "rename", + "repeat", + "reshape", + "result", + "return", + "rewind", + "rewrite", + "rrspacing", + "rshift", + "same_type_as", + "save", + "scale", + "scan", + "secnds", + "second", + "sections", + "select", + "selected_char_kind", + "selected_int_kind", + "selected_real_kind", + "sequence", + "sequential", + "set_exponent", + "shape", + "shared", + "shifta", + "shiftl", + "shiftr", + "short", + "sign", + "signal", + "sin", + "sind", + "sinh", + "size", + "sizeof", + "sleep", + "sngl", + "source", + "spacing", + "spread", + "sqrt", + "srand", + "stat", + "stat_failed_image", + "stat_locked", + "stat_locked_other_image", + "stat_stopped_image", + "stat_unlocked", + "status", + "stop", + "storage_size", + "structure", + "submodule", + "subroutine", + "sum", + "symlnk", + "sync", + "system", + "system_clock", + "tan", + "tand", + "tanh", + "target", + "task", + "taskwait", + "then", + "this_image", + "threadprivate", + "time", + "time8", + "tiny", + "to", + "trailz", + "transfer", + "transpose", + "trim", + "true", + "ttynam", + "type", + "ubound", + "ucobound", + "umask", + "unformatted", + "unit", + "unlink", + "unlock", + "unpack", + "use", + "value", + "verif", + "verify", + "volatile", + "wait", + "where", + "while", + "workshare", + "write", + "xor", + "zabs", + "zcos", + "zexp", + "zlog", + "zsin", + "zsqrt", +} CODE_REPLACEMENTS = [ # Replace Fortran 77 style conditional keywords - (r'\.eq\.', ' == '), - (r'\.ne\.', ' /= '), - (r'\.gt\.', ' > '), - (r'\.lt\.', ' < '), - (r'\.ge\.', ' >= '), - (r'\.le\.', ' <= '), + (r"\.eq\.", " == "), + (r"\.ne\.", " /= "), + (r"\.gt\.", " > "), + (r"\.lt\.", " < "), + (r"\.ge\.", " >= "), + (r"\.le\.", " <= "), # protect 'operator' definitions e.g. "OPERATOR(/)", from the array # initiialisation enforcement by enforcing spaces around the operators. # Make all operators follow the same style but only (/) and (/=) had issues. - (r'\(\s*\*\s*\)', '( * )'), - (r'\(\s*\+\s*\)', '( + )'), - (r'\(\s*-\s*\)', '( - )'), - (r'\(\s*\/\s*\)', '( / )'), - (r'\(\s*==\s*\)', '( == )'), - (r'\(\s*\/=\s*\)', '( /= )'), - (r'\(\s*<\s*\)', '( < )'), - (r'\(\s*<=\s*\)', '( <= )'), - (r'\(\s*>\s*\)', '( > )'), - (r'\(\s*>=\s*\)', '( >= )'), + (r"\(\s*\*\s*\)", "( * )"), + (r"\(\s*\+\s*\)", "( + )"), + (r"\(\s*-\s*\)", "( - )"), + (r"\(\s*\/\s*\)", "( / )"), + (r"\(\s*==\s*\)", "( == )"), + (r"\(\s*\/=\s*\)", "( /= )"), + (r"\(\s*<\s*\)", "( < )"), + (r"\(\s*<=\s*\)", "( <= )"), + (r"\(\s*>\s*\)", "( > )"), + (r"\(\s*>=\s*\)", "( >= )"), # Replace array initialisations - (r'\(\/', '['), - (r'\/\)', ']'), + (r"\(\/", "["), + (r"\/\)", "]"), # Ensure remaining comparitive logicals have spaces either side - (r'([^\s])(? .not.'), - (r'\.not\.([^\s])', r'.not. \g<1>'), - (r'([^\s])\.and\.', r'\g<1> .and.'), - (r'\.and\.([^\s])', r'.and. \g<1>'), - (r'([^\s])\.or\.', r'\g<1> .or.'), - (r'\.or\.([^\s])', r'.or. \g<1>'), - (r'([^\s])\.eqv\.', r'\g<1> .eqv.'), - (r'\.eqv\.([^\s])', r'.eqv. \g<1>'), - (r'([^\s])\.neqv\.', r'\g<1> .neqv.'), - (r'\.neqv\.([^\s])', r'.neqv. \g<1>'), + (r"([^\s])(? .not."), + (r"\.not\.([^\s])", r".not. \g<1>"), + (r"([^\s])\.and\.", r"\g<1> .and."), + (r"\.and\.([^\s])", r".and. \g<1>"), + (r"([^\s])\.or\.", r"\g<1> .or."), + (r"\.or\.([^\s])", r".or. \g<1>"), + (r"([^\s])\.eqv\.", r"\g<1> .eqv."), + (r"\.eqv\.([^\s])", r".eqv. \g<1>"), + (r"([^\s])\.neqv\.", r"\g<1> .neqv."), + (r"\.neqv\.([^\s])", r".neqv. \g<1>"), # Ensure hard-coded real numbers have a zero after the decimal point - (r'([0-9])\.([^0-9]|$)', r'\g<1>.0\g<2>'), + (r"([0-9])\.([^0-9]|$)", r"\g<1>.0\g<2>"), # Remove start of line ampersands, without changing the spacing of # the line they appear on - (r'^(\s*)&(.*\w.*)$', r'\g<1> \g<2>'), + (r"^(\s*)&(.*\w.*)$", r"\g<1> \g<2>"), # Make constructs which include brackets have exactly one space # between the construct and the bracket character - (r'(^\s*)(\w+\s*:\s*|[0-9]+\s*|)((else|)\s*if)(|\s\s+)\(', - r'\g<1>\g<2>\g<3> ('), - (r'(^\s*)(\w+\s*:\s*|[0-9]+\s*|)where(|\s\s+)\(', r'\g<1>\g<2>where ('), - (r'(^\s*)case(|\s\s+)\(', r'\g<1>case ('), - (r'\)(|\s\s+)then(\W|$)', r') then\g<1>'), + (r"(^\s*)(\w+\s*:\s*|[0-9]+\s*|)((else|)\s*if)(|\s\s+)\(", r"\g<1>\g<2>\g<3> ("), + (r"(^\s*)(\w+\s*:\s*|[0-9]+\s*|)where(|\s\s+)\(", r"\g<1>\g<2>where ("), + (r"(^\s*)case(|\s\s+)\(", r"\g<1>case ("), + (r"\)(|\s\s+)then(\W|$)", r") then\g<1>"), # Make intent statements contain no extra spacing inside the brackets - (r'(.*intent\s*)\(\s*in\s*\)(.*)', r'\g<1>(in)\g<2>'), - (r'(.*intent\s*)\(\s*out\s*\)(.*)', r'\g<1>(out)\g<2>'), - (r'(.*intent\s*)\(\s*in out\s*\)(.*)', r'\g<1>(in out)\g<2>'), + (r"(.*intent\s*)\(\s*in\s*\)(.*)", r"\g<1>(in)\g<2>"), + (r"(.*intent\s*)\(\s*out\s*\)(.*)", r"\g<1>(out)\g<2>"), + (r"(.*intent\s*)\(\s*in out\s*\)(.*)", r"\g<1>(in out)\g<2>"), # Make module USE, ONLY statments have exactly no space between the ONLY # and the colon character after it - (r'^(\s*)use(\s*,\s*\w+\s*::|)(\s+\w+\s*,\s*)only\s*:(.*)$', - r'\g<1>use\g<2>\g<3>only:\g<4>'), + ( + r"^(\s*)use(\s*,\s*\w+\s*::|)(\s+\w+\s*,\s*)only\s*:(.*)$", + r"\g<1>use\g<2>\g<3>only:\g<4>", + ), ] COMMENT_REPLACEMENTS = [ # DEPENDS ON fcm constructions - (r'^(\s*!)\s*depends\s*on\s*:\s*', r'\g<1> DEPENDS ON: '), + (r"^(\s*!)\s*depends\s*on\s*:\s*", r"\g<1> DEPENDS ON: "), ] FORTRAN_TYPES = [ diff --git a/suite_report.py b/suite_report.py index 29712d05..0f006d3a 100755 --- a/suite_report.py +++ b/suite_report.py @@ -30,21 +30,23 @@ from __future__ import print_function import glob +import json import os import re import sqlite3 +import subprocess import sys -import traceback import time -import subprocess -import json -from argparse import ArgumentParser, ArgumentTypeError, \ - RawDescriptionHelpFormatter +import traceback +from argparse import ArgumentParser, ArgumentTypeError, RawDescriptionHelpFormatter from collections import defaultdict +from optparse import OptionGroup, OptionParser + from fcm_bdiff import get_branch_diff_filenames try: import argcomplete + COMPLETION = True except ModuleNotFoundError: COMPLETION = False @@ -181,9 +183,7 @@ def _read_file(filename): lines = filehandle.readlines() else: print(f'[ERROR] Unable to find file :\n "{filename}"') - raise IOError( - f'_read_file got invalid filename : "{filename}"' - ) + raise IOError(f'_read_file got invalid filename : "{filename}"') return lines @@ -205,8 +205,10 @@ def _run_command(command, ignore_fail=False): Returns the exit code, standard out and standard error as list. """ with subprocess.Popen( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding="utf-8", + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", ) as pobj: pobj.wait() retcode, stdout, stderr = ( @@ -349,6 +351,7 @@ def _parse_string( # pylint: disable=too-many-branches # pylint: disable=too-many-public-methods + class SuiteReport: """Object to hold data and methods required to produce a suite report from a rose-stem suite output.""" @@ -367,9 +370,7 @@ def __init__( status when generating the task table in the report. """ self.suite_path = os.path.abspath(suite_path) - self.is_cylc8 = os.path.isdir( - os.path.join(self.suite_path, "log", "config") - ) + self.is_cylc8 = os.path.isdir(os.path.join(self.suite_path, "log", "config")) self.log_path = log_path self.sort_by_name = sort_by_name @@ -390,8 +391,7 @@ def __init__( # Resolve "runN" soft link - Required for Cylc8 # cylc-review path link_target = os.readlink(self.suite_path) - suitename = os.path.join(os.path.dirname(self.suite_path), - link_target) + suitename = os.path.join(os.path.dirname(self.suite_path), link_target) except OSError: suitename = self.suite_path @@ -399,8 +399,7 @@ def __init__( # Default to userID from suite path unless CYLC_SUITE_OWNER is # present self.suite_owner = os.environ.get( - "CYLC_SUITE_OWNER", - os.path.basename(suite_dir.rstrip("/")) + "CYLC_SUITE_OWNER", os.path.basename(suite_dir.rstrip("/")) ) self.parse_rose_suite_run() @@ -428,9 +427,7 @@ def __init__( invalid = [] for project in self.job_sources: proj_dict = self.job_sources[project] - proj_dict["tested source"] = _remove_quotes( - proj_dict["tested source"] - ) + proj_dict["tested source"] = _remove_quotes(proj_dict["tested source"]) if "repo loc" in proj_dict: proj_dict["repo loc"] = self.convert_to_srs( proj_dict["repo loc"], self.projects @@ -465,13 +462,9 @@ def __init__( if ":" in url and "@" not in url: revision = _get_current_head_revision(mirror_url, fcm_exec) proj_dict[location + " loc"] = url + "@" + revision - proj_dict[location + " mirror"] = ( - mirror_url + "@" + revision - ) + proj_dict[location + " mirror"] = mirror_url + "@" + revision proj_dict["repo link"] = self.generate_link(proj_dict["repo loc"]) - proj_dict["parent link"] = self.generate_link( - proj_dict["parent loc"] - ) + proj_dict["parent link"] = self.generate_link(proj_dict["parent loc"]) # If those attempts to generate links didn't work, try the hope # and guess approach. if proj_dict["repo link"] is None: @@ -517,8 +510,7 @@ def __init__( self.only_common_groups = True else: self.only_common_groups = all( - group.strip() in COMMON_GROUPS[self.site] - for group in self.groups + group.strip() in COMMON_GROUPS[self.site] for group in self.groups ) # Finally, remove any projects which were deemed invalid. @@ -528,8 +520,7 @@ def __init__( def debug_print_obj(self): """Debug print method. Prints everything in the SuiteReport object.""" - print("-" * 80 + "\nSet up SuiteReport object\n" - + "-" * 80 + "\n\n") + print("-" * 80 + "\nSet up SuiteReport object\n" + "-" * 80 + "\n\n") for key, value in self.__dict__.items(): if key == "projects": print(f'{key} contains "{len(value)}" entries.') @@ -540,16 +531,11 @@ def debug_print_obj(self): elif key == "verbosity": text = "Verbosity level is set to : " if value >= 4: - print( - text - + "Hide Housekeeping, Gatekeeping and Successful tasks" - ) + print(text + "Hide Housekeeping, Gatekeeping and Successful tasks") elif value >= 3: print( - text - + "Hide Housekeeping, Gatekeeping and if all " - + "groups run were \"common\" groups also hide " - + "Successful tasks" + text + "Hide Housekeeping, Gatekeeping and if all groups run " + 'were "common" groups also hide Successful tasks' ) elif value >= 2: print(text + "Hide Housekeeping and Gatekeeping tasks") @@ -561,9 +547,7 @@ def debug_print_obj(self): self.print_job_sources(value) else: print(f'{key} is :"{value}"') - print( - "\n" + "-" * 80 + "\nEnd of SuiteReport object\n" + "-" * 80 + "\n" - ) + print("\n" + "-" * 80 + "\nEnd of SuiteReport object\n" + "-" * 80 + "\n") @staticmethod def print_job_sources(job_srcs_dict): @@ -599,8 +583,7 @@ def parse_processed_config_file(self): break else: sys.exit( - "Error: Couldn't find a *-rose-suite.conf file in " - + f"{srp_file}" + "Error: Couldn't find a *-rose-suite.conf file in " + f"{srp_file}" ) else: srp_file = os.path.join(suite_dir, PROCESSED_SUITE_RC) @@ -632,13 +615,11 @@ def parse_processed_config_file(self): sources[result.group(1)] = {} if " " in result.group(3): multiple_branches[(result.group(1))] = result.group(3) - sources[result.group(1)][ - "tested source" - ] = result.group(3).split()[0] + sources[result.group(1)]["tested source"] = result.group( + 3 + ).split()[0] else: - sources[result.group(1)][ - "tested source" - ] = result.group(3) + sources[result.group(1)]["tested source"] = result.group(3) self.rose_orig_host = rose_orig_host self.job_sources = sources @@ -713,11 +694,8 @@ def initialise_projects(self): # Check for keywords conforming to the meto prescribed pattern # of ending in '.x' for the external repo and '.xm' for the # local mirror. - if ( - find_x_keyword.search(project) and find_srs_url.match(url) - ) or ( - find_xm_keyword.search(project) - and find_mirror_url.match(url) + if (find_x_keyword.search(project) and find_srs_url.match(url)) or ( + find_xm_keyword.search(project) and find_mirror_url.match(url) ): projects[project] = url self.projects = projects @@ -729,9 +707,7 @@ def cylc7_check_versions_file(self, projects): """ find_proj_name = re.compile(r"/(\w+)-\d+.version") version_files = [] - version_files = glob.glob( - f"{self.suite_path}/log/*.version" - ) + version_files = glob.glob(f"{self.suite_path}/log/*.version") for vfile in version_files: if "rose-suite-run.version" in vfile: @@ -792,9 +768,9 @@ def check_versions_files(self): prefix = "https://code.metoffice.gov.uk/svn/" prefix_svn = "svn://fcm1/" if project.startswith(prefix): - project = project[len(prefix):] + project = project[len(prefix) :] if project.startswith(prefix_svn): - project = project[len(prefix_svn):] + project = project[len(prefix_svn) :] project = re.split("[/.]", project)[0].upper() projects[project] = {} @@ -936,9 +912,7 @@ def generate_owner_dictionary(self, mode): file_path = self.export_file("fcm:um.xm_tr", fname, exported_file) if file_path is None: # Couldn't check out file - use working copy Owners file instead - wc_path = get_working_copy_path( - self.job_sources["UM"]["tested source"] - ) + wc_path = get_working_copy_path(self.job_sources["UM"]["tested source"]) if not wc_path: wc_path = "" file_path = os.path.join(wc_path, fname) @@ -969,9 +943,7 @@ def generate_owner_dictionary(self, mode): others = "" except IndexError: others = "" - owners_dict.update( - {section.lower(): [owners, others]} - ) + owners_dict.update({section.lower(): [owners, others]}) except EnvironmentError: print("Can't find working copy for Owners File") return None @@ -1003,9 +975,7 @@ def create_approval_table(needed_approvals, mode): if needed_approvals is None: table += [ - " |||||| No UM " - + mode.capitalize() - + " Owner Approvals Required || " + " |||||| No UM " + mode.capitalize() + " Owner Approvals Required || " ] else: for owner in needed_approvals.keys(): @@ -1080,8 +1050,7 @@ def required_config_approvals(self, failed_configs): if config_owners is None: return None - config_approvals = self.get_config_owners(failed_configs, - config_owners) + config_approvals = self.get_config_owners(failed_configs, config_owners) if len(config_approvals.keys()) == 0: config_approvals = None @@ -1160,7 +1129,10 @@ def get_code_owners(self, code_owners): # Couldn't check out working copy file - deleted? Use trunk instead file_path = self.export_file("fcm:um.xm_tr", fpath) if file_path is None: - print('[WARN] Unable to establish code section for file: ', fle) + print( + "[WARN] Unable to establish code section for file: ", + fle, + ) file_path = "" try: @@ -1326,9 +1298,7 @@ def check_lfric_extract_list(self): if extract_list_path: try: - extract_list_dict = self.parse_lfric_extract_list( - extract_list_path - ) + extract_list_dict = self.parse_lfric_extract_list(extract_list_path) except (EnvironmentError, TypeError, AttributeError): # Potential error here changed type between python2 and 3 extract_list_path = None @@ -1406,9 +1376,7 @@ def forced_status_sort(item_tuple): find_monitor = re.compile(r"monitor") find_gatekeeper = re.compile(r"gatekeeper") failed_configs = [] - for task, state in sorted( - list(data.items()), key=key_by_name_or_status - ): + for task, state in sorted(list(data.items()), key=key_by_name_or_status): # Count the number of times task have any given status. status_counts[state] += 1 if (verbosity >= 1) and find_housekeep.match(task): @@ -1435,9 +1403,7 @@ def forced_status_sort(item_tuple): # Check if task requires extra care for extra_care_string in HIGHLIGHT_ROSE_ANA_FAILS: if extra_care_string in task: - highlight_start = ( - "'''[[span(style=color: #FF00FF, *****" - ) + highlight_start = "'''[[span(style=color: #FF00FF, *****" highlight_end = "***** )]]'''" status_counts[PINK_FAIL_TEXT] += 1 status_counts[state] -= 1 @@ -1446,13 +1412,10 @@ def forced_status_sort(item_tuple): # Record this as a failed config failed_configs.append(task) - lines.append( - f" || {task} || {highlight_start}{state}{highlight_end} || " - ) + lines.append(f" || {task} || {highlight_start}{state}{highlight_end} || ") if len(lines) == 1: lines.append( - " |||| This table is deliberately empty as all tasks " - "are hidden || " + " |||| This table is deliberately empty as all tasks " "are hidden || " ) status_summary = ["'''Suite Output'''"] @@ -1460,29 +1423,20 @@ def forced_status_sort(item_tuple): status_summary += [""] status_summary.append(" |||| '''All Tasks''' || ") status_summary.append(" || '''Status''' || '''No. of Tasks''' || ") - for status, count in sorted( - status_counts.items(), key=forced_status_sort - ): - status_summary.append( - f" || {status} || {count} || " - ) + for status, count in sorted(status_counts.items(), key=forced_status_sort): + status_summary.append(f" || {status} || {count} || ") status_summary.append("") if len(hidden_counts) > 0: status_summary.append(" |||| '''Hidden Tasks''' || ") - status_summary.append( - " || '''Type''' || '''No. of Tasks Hidden''' || " - ) + status_summary.append(" || '''Type''' || '''No. of Tasks Hidden''' || ") for task_type, count in hidden_counts.items(): - status_summary.append( - f" || {task_type} || {count} || " - ) + status_summary.append(f" || {task_type} || {count} || ") status_summary.append("") # Check whether lfric shared files have been touched # Not needed if lfric the suite source lfric_testing_message = [""] - if ("LFRIC" not in self.primary_project - and self.primary_project != "UNKNOWN"): + if "LFRIC" not in self.primary_project and self.primary_project != "UNKNOWN": lfric_testing_message = self.check_lfric_extract_list() # Generate table for required config and code owners @@ -1492,9 +1446,7 @@ def forced_status_sort(item_tuple): co_approval_table = self.required_co_approvals() if co_approval_table: return_list += co_approval_table - config_approval_table = self.required_config_approvals( - failed_configs - ) + config_approval_table = self.required_config_approvals(failed_configs) if config_approval_table: return_list += config_approval_table return lfric_testing_message + return_list + status_summary + lines @@ -1520,9 +1472,7 @@ def convert_to_mirror(url, projects_dict): if new_proj in projects_dict: old_proj_url = proj_url new_proj_url = projects_dict[new_proj] - mirror_url = re.sub( - old_proj_url, new_proj_url, url, count=1 - ) + mirror_url = re.sub(old_proj_url, new_proj_url, url, count=1) break # checking given url against keywords in the projects_dict elif proj in url: @@ -1575,8 +1525,7 @@ def convert_to_srs(url, projects_dict): ) # maintain keyword style, but convert to srs. else: - srs_url = re.sub(proj, shared_project, url, - count=1) + srs_url = re.sub(proj, shared_project, url, count=1) break return srs_url @@ -1717,9 +1666,7 @@ def gen_table_element(text_list, link, bold=False): ) if "ticket no" in proj_dict: if proj_dict["ticket no"] is not None: - project_ticket_link = [ - f"{project}:{proj_dict['ticket no']}" - ] + project_ticket_link = [f"{project}:{proj_dict['ticket no']}"] else: project_ticket_link = [None] else: @@ -1763,9 +1710,7 @@ def gen_resources_table(self): + "'''Total Memory''' ||", ] found_nothing = False - lines.append( - f" || {job} || {wallclock} || {memory} || " - ) + lines.append(f" || {job} || {wallclock} || {memory} || ") lines.append("") return lines @@ -1775,15 +1720,11 @@ def get_wallclock_and_memory(filename): and memory.""" wallclock = "Unavailable" memory = "Unavailable" - find_wallclock = re.compile( - r"PE\s*0\s*Elapsed Wallclock Time:\s*(\d+(\.\d+|))" - ) + find_wallclock = re.compile(r"PE\s*0\s*Elapsed Wallclock Time:\s*(\d+(\.\d+|))") find_total_mem = re.compile(r"Total Mem\s*(\d+)") find_um_atmos_exe = re.compile(r"um-atmos.exe") check_for_percentage = re.compile("[0-9]+[%]") - find_mem_n_units = re.compile( - r"(?P[0-9]*\.[0-9]*)(?P[A-Za-z])" - ) + find_mem_n_units = re.compile(r"(?P[0-9]*\.[0-9]*)(?P[A-Za-z])") # pylint: disable=broad-exception-caught @@ -1810,9 +1751,7 @@ def get_wallclock_and_memory(filename): wallclock = "Failure processing EOJ" memory = "Failure processing EOJ" stacktr = traceback.format_exc() - print( - f"[ERROR] Processing wallclock and memory use :\n{stacktr}" - ) + print(f"[ERROR] Processing wallclock and memory use :\n{stacktr}") print(f"Error type : {type(err)}") print(err) @@ -1865,17 +1804,12 @@ def get_altered_files_list(mirror_loc): try: # Get a list of altered files from the fcm mirror url - bdiff_files = get_branch_diff_filenames( - mirror_loc, path_override="" - ) + bdiff_files = get_branch_diff_filenames(mirror_loc, path_override="") break except Exception as err: print(err) if attempt == 4: - print( - "Cant get list of alterered files - returning " - "empty list." - ) + print("Cant get list of alterered files - returning " "empty list.") bdiff_files = [] break @@ -1926,53 +1860,36 @@ def print_report(self): except KeyError: pass bg_colour = BACKGROUND_COLOURS[self.primary_project.lower()] - trac_log.append( - f"{{{{{{#!div style='background : {bg_colour}'" - ) + trac_log.append(f"{{{{{{#!div style='background : {bg_colour}'") if ticket_nos != "": trac_log.append( - f" = Ticket {ticket_nos} " - + "Testing Results - rose-stem output = " + f" = Ticket {ticket_nos} " + "Testing Results - rose-stem output = " ) else: trac_log.append(" = Testing Results - rose-stem output = ") trac_log.append("") - trac_log.append( - f" || Suite Name: || {self.suitename} || " - ) + trac_log.append(f" || Suite Name: || {self.suitename} || ") - trac_log.append( - f" || Suite Owner: || {self.suite_owner} || " - ) + trac_log.append(f" || Suite Owner: || {self.suite_owner} || ") if self.trustzone: - trac_log.append( - f" || Trustzone: || {self.trustzone} || " - ) + trac_log.append(f" || Trustzone: || {self.trustzone} || ") if self.fcm: - trac_log.append( - f" || FCM version: || {self.fcm} || " - ) + trac_log.append(f" || FCM version: || {self.fcm} || ") if self.rose: - trac_log.append( - f" || Rose version: || {self.rose} || " - ) + trac_log.append(f" || Rose version: || {self.rose} || ") if self.cylc: - trac_log.append( - f" || Cylc version: || {self.cylc} || " - ) + trac_log.append(f" || Cylc version: || {self.cylc} || ") - trac_log.append( - f" || Report Generated: || {self.creation_time} || " - ) + trac_log.append(f" || Report Generated: || {self.creation_time} || ") - review_url = os.path.join(CYLC_REVIEW_URL[self.site], - "taskjobs", - self.suite_owner) + review_url = os.path.join( + CYLC_REVIEW_URL[self.site], "taskjobs", self.suite_owner + ) trac_log.append( f" || Cylc-Review: || {review_url}/?suite={self.suitename} || " ) @@ -1982,9 +1899,7 @@ def print_report(self): f" || Groups Run: || {self.generate_groups(self.groups)} || " ) if self.rose_orig_host is not None: - trac_log.append( - f" || ''ROSE_ORIG_HOST:'' || {self.rose_orig_host} || " - ) + trac_log.append(f" || ''ROSE_ORIG_HOST:'' || {self.rose_orig_host} || ") if self.host_xcs: trac_log.append(" || HOST_XCS || True || ") trac_log.append("") @@ -2006,10 +1921,7 @@ def print_report(self): trac_log.append("-----") trac_log.append("") - if ( - not self.required_comparisons - and "LFRIC_APPS" not in self.job_sources - ): + if not self.required_comparisons and "LFRIC_APPS" not in self.job_sources: trac_log.append("") trac_log.append("-----") trac_log.append(" = WARNING !!! = ") @@ -2046,9 +1958,7 @@ def print_report(self): db_file = "" if self.is_cylc8: - db_file = os.path.join( - self.suite_path, "log", SUITE_DB_FILENAME_CYLC8 - ) + db_file = os.path.join(self.suite_path, "log", SUITE_DB_FILENAME_CYLC8) else: db_file = os.path.join(self.suite_path, SUITE_DB_FILENAME) @@ -2074,8 +1984,7 @@ def print_report(self): suite_dir = "--cylc_suite_dir--" trac_log.extend( [ - "There has been an exception in " - + "SuiteReport.print_report()", + "There has been an exception in SuiteReport.print_report()", "See output for more information", "rose-stem suite output will be in the files :\n", f"~/cylc-run/{suite_dir}/log/suite/log", @@ -2095,12 +2004,9 @@ def print_report(self): try: _write_file(trac_log_path, trac_log, newline=True) except IOError: + print(f"[ERROR] Writing to {TRAC_LOG_FILE} file : {trac_log_path}") print( - f"[ERROR] Writing to {TRAC_LOG_FILE} file : {trac_log_path}" - ) - print( - f"{TRAC_LOG_FILE} to this point " - + "would have read as follows :\n" + f"{TRAC_LOG_FILE} to this point " + "would have read as follows :\n" ) print(f"----- Start of {TRAC_LOG_FILE}.log -----") for line in trac_log: @@ -2111,6 +2017,7 @@ def print_report(self): # pylint: enable=broad-exception-caught + # pylint: enable=too-many-instance-attributes # pylint: enable=too-many-locals # pylint: enable=too-many-statements @@ -2141,7 +2048,6 @@ def get_working_copy_path(path): def directory_type(opt): - """Check location exists and is a directory.""" if not os.path.exists(opt): @@ -2155,59 +2061,75 @@ def directory_type(opt): def parse_arguments(): - """Process command line arguments.""" suite_path = os.environ.get( # Cylc7 environment variable "CYLC_SUITE_RUN_DIR", # Default to Cylc8 environment variable - os.environ.get("CYLC_WORKFLOW_RUN_DIR", None) + os.environ.get("CYLC_WORKFLOW_RUN_DIR", None), ) - parser = ArgumentParser(usage="%(prog)s [options] [args]", - description="Generate a suite report", - formatter_class=RawDescriptionHelpFormatter) + parser = ArgumentParser( + usage="%(prog)s [options] [args]", + description="Generate a suite report", + formatter_class=RawDescriptionHelpFormatter, + ) paths = parser.add_argument_group("location arguments") - item = paths.add_argument("-S", "--suite-path", - type=directory_type, - dest="suite_path", - metavar="DIR", - default=suite_path, - help="path to suite") + item = paths.add_argument( + "-S", + "--suite-path", + type=directory_type, + dest="suite_path", + metavar="DIR", + default=suite_path, + help="path to suite", + ) if COMPLETION: item.completer = argcomplete.DirectoriesCompleter() - item = paths.add_argument("-L", "--log_path", type=directory_type, - dest="log_path", - metavar="DIR", - help=f"output dir for {TRAC_LOG_FILE}") + item = paths.add_argument( + "-L", + "--log_path", + type=directory_type, + dest="log_path", + metavar="DIR", + help=f"output dir for {TRAC_LOG_FILE}", + ) if COMPLETION: item.completer = argcomplete.DirectoriesCompleter() verbose = parser.add_argument_group("diagnostic arguments") - verbose.add_argument("-v", "--increase-verbosity", - dest="increase_verbosity", - action="count", - default=0, - help="increases Verbosity level. " - f"(default: {DEFAULT_VERBOSITY})") + verbose.add_argument( + "-v", + "--increase-verbosity", + dest="increase_verbosity", + action="count", + default=0, + help="increases Verbosity level. " f"(default: {DEFAULT_VERBOSITY})", + ) - verbose.add_argument("-q", "--decrease-verbosity", - dest="decrease_verbosity", - action="count", - default=0, - help="decreases Verbosity level.") + verbose.add_argument( + "-q", + "--decrease-verbosity", + dest="decrease_verbosity", + action="count", + default=0, + help="decreases Verbosity level.", + ) misc = parser.add_argument_group("misc arguments") - misc.add_argument("-N", "--name-sort", - dest="sort_by_name", - action="store_true", - help="sort task table by task names") + misc.add_argument( + "-N", + "--name-sort", + dest="sort_by_name", + action="store_true", + help="sort task table by task names", + ) opts, rest = parser.parse_known_args() diff --git a/tests/test_fcm_bdiff.py b/tests/test_fcm_bdiff.py index 1808b0b6..a22d6449 100644 --- a/tests/test_fcm_bdiff.py +++ b/tests/test_fcm_bdiff.py @@ -1,4 +1,5 @@ import pytest + from fcm_bdiff import * # Use Case Testing for get_branch_diff_filenames @@ -20,20 +21,23 @@ # Use Case 2 - Test changes made to a working copy, # use url of the branch as the first parameter # svn/um/main variant -use_case_2_1_branch = 'https://code.metoffice.gov.uk/svn/um/main/branches/'\ - + 'dev/Share/r118291_fcm_bdiff_share_testing_branch' +use_case_2_1_branch = ( + "https://code.metoffice.gov.uk/svn/um/main/branches/" + + "dev/Share/r118291_fcm_bdiff_share_testing_branch" +) use_case_2_1_expected = [ - 'https://code.metoffice.gov.uk/svn/um/main/branches/' - + 'dev/Share/r118291_fcm_bdiff_share_testing_branch/src/atmosphere/' - + 'AC_assimilation/ac2.F90' - ] + "https://code.metoffice.gov.uk/svn/um/main/branches/" + + "dev/Share/r118291_fcm_bdiff_share_testing_branch/src/atmosphere/" + + "AC_assimilation/ac2.F90" +] # trac/um/browser variant -use_case_2_2_branch = 'fcm:um.x_br/'\ - + 'dev/Share/r118291_fcm_bdiff_share_testing_branch' +use_case_2_2_branch = ( + "fcm:um.x_br/" + "dev/Share/r118291_fcm_bdiff_share_testing_branch" +) use_case_2_2_expected = [ - 'fcm:um.x_br/dev/Share/r118291_fcm_bdiff_share_testing_branch/src/' - + 'atmosphere/AC_assimilation/ac2.F90' - ] + "fcm:um.x_br/dev/Share/r118291_fcm_bdiff_share_testing_branch/src/" + + "atmosphere/AC_assimilation/ac2.F90" +] # Use Case 3 - Test changes made to a working copy, @@ -51,8 +55,8 @@ ("../../_svn/main/trunk", True), # PASS ("../incorrect_value/trunk", False), # PASS ("..*_svn/main/trunk", True), # PASS - ("..*incorrect_value/trunk", False) # PASS - ] + ("..*incorrect_value/trunk", False), # PASS + ], ) def test_is_trunk(url, expected): assert is_trunk(url) == expected @@ -61,15 +65,9 @@ def test_is_trunk(url, expected): @pytest.mark.parametrize( ("bytes_type_string", "expected"), [ - ( - bytes("my_test_string", encoding='utf-8'), - "my_test_string" - ), # PASS - ( - bytes("my_test_string", encoding='cp1252'), - "my_test_string" - ) # PASS - ] + (bytes("my_test_string", encoding="utf-8"), "my_test_string"), # PASS + (bytes("my_test_string", encoding="cp1252"), "my_test_string"), # PASS + ], ) def test_text_decoder(bytes_type_string, expected): assert text_decoder(bytes_type_string) == expected @@ -80,8 +78,8 @@ def test_text_decoder(bytes_type_string, expected): [ # Use Case 2 (use_case_2_1_branch, use_case_2_1_expected), # PASS - (use_case_2_2_branch, use_case_2_2_expected) # PASS - ] + (use_case_2_2_branch, use_case_2_2_expected), # PASS + ], ) def test_get_branch_diff_filenames(branch, expected): assert get_branch_diff_filenames(branch) == expected