diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 653a6393..42ef575a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: "3.12" - name: Install dependencies run: | @@ -41,7 +41,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version: '24' + node-version: "24" - name: Install dependencies run: npm ci @@ -62,7 +62,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version: '24' + node-version: "24" - name: Ensure pip cache directory exists run: mkdir -p ~/.cache/pip @@ -101,6 +101,7 @@ jobs: bash scripts/install.sh - name: Initialize Browser library + timeout-minutes: 5 run: | rfbrowser init diff --git a/docs/basic-command-line-interface-cli.md b/docs/basic-command-line-interface-cli.md index 589f8dcb..57d42549 100644 --- a/docs/basic-command-line-interface-cli.md +++ b/docs/basic-command-line-interface-cli.md @@ -116,6 +116,11 @@ robotdashboard -r alias=some_cool_alias,tag=prod,tag=dev -r alias=alias12345 robotdashboard -r limit=10 robotdashboard -r age=10d # (y)ear/(d)ay/(h)our/(m)inute/(s)econd supported robotdashboard -r age=-10d +# Log data of removed runs in jsonl +robotdashboard -r limit=10 --logremoved "/myLogDir/removedRuns.jsonl" +robotdashboard -r limit=10 --logremoved run:suite:"/myLogDir/removedRuns.jsonl" +robotdashboard -r limit=10 --logremoved all +robotdashboard -r limit=10 --logremoved run:keyword ``` - Optional: `-r` or `--removeruns` specifies one or more runs to remove. - Multiple values are separated by commas (,). @@ -125,6 +130,11 @@ robotdashboard -r age=-10d - With limit=10 only the 10 most recent runs will be kept, all others will be removed. - With age=10d only runs _**older**_ than 10 days will be removed - With age=-10d only runs _**younger**_ than 10 days will be removed +- Optional: `--logremoved` logs run data to a `.jsonl` file before removal. +- Format: `[types:]path` where types are colon-separated from `run`, `suite`, `test`, `keyword`, `all`. +- If no types are specified, defaults to `all` (runs, suites, tests and keywords). +- If no path is provided, a timestamped file `robot_removed_runs_YYYYMMDD-HHMMSS.jsonl` is created in the current directory. +- Each removed run is appended as one JSON line containing the selected data types. ### Use a custom database class ```bash diff --git a/robotframework_dashboard/arguments.py b/robotframework_dashboard/arguments.py index 75a90355..77f63e02 100644 --- a/robotframework_dashboard/arguments.py +++ b/robotframework_dashboard/arguments.py @@ -4,8 +4,16 @@ from re import split from os import getcwd from os.path import join, exists +from typing import NamedTuple from .version import __version__ +_VALID_LOG_TYPES = {"run", "suite", "test", "keyword", "all"} + + +class LogRemovedConfig(NamedTuple): + types: list + path: str + class dotdict(dict): """dot.notation access to dictionary attributes""" @@ -275,6 +283,22 @@ def _parse_arguments(self): nargs="*", default=None, ) + db_group.add_argument( + "--logremoved", + metavar="TYPES_PATH", + help=( + "Log (append if file exists) run data to jsonl before removing from the db\n" + " • Supply what types to log (default=all) and (optional) a path to the desired .jsonl\n" + "Examples:\n" + " • '--logremoved \"/tmp/removed_runs.jsonl\"'\n" + " • '--logremoved \"run:suite\"'\n" + " • '--logremoved \"all:/tmp/removed_runs.jsonl\"'\n" + ), + action="store", + type=str, + default=None, + dest="logremoved", + ) db_group.add_argument( "-c", "--databaseclass", @@ -605,6 +629,23 @@ def _process_arguments(self, arguments): ssl_certfile = arguments.ssl_certfile ssl_keyfile = arguments.ssl_keyfile + # handles --logremoved: "type1:type2:path" or "path" or "type1:type2" + log_removed = None + if arguments.logremoved: + parts = arguments.logremoved.split(":") + types, remaining = [], list(parts) + # grab element types, leave path, for windows drive letter X: parse + for part in parts: + if part in _VALID_LOG_TYPES: + types.append(part) + remaining.pop(0) + else: + break + if not types: + types = ["all"] + path = ":".join(remaining) if remaining else f"robot_removed_runs_{generation_datetime.strftime('%Y%m%d-%H%M%S')}.jsonl" + log_removed = LogRemovedConfig(types=types, path=path) + # validates argument combinations self._check_argument_errors(arguments, outputs, outputfolderpaths, force_json_config, database_class) self._check_argument_warnings(arguments, outputs, outputfolderpaths, use_logs, generate_dashboard, no_autoupdate, offline_dependencies) @@ -640,5 +681,6 @@ def _process_arguments(self, arguments): "ssl_certfile": ssl_certfile, "ssl_keyfile": ssl_keyfile, "log_url": arguments.logurl, + "log_removed": log_removed, } return dotdict(provided_args) diff --git a/robotframework_dashboard/database.py b/robotframework_dashboard/database.py index 220ec55b..f7265d70 100644 --- a/robotframework_dashboard/database.py +++ b/robotframework_dashboard/database.py @@ -6,6 +6,7 @@ from time import time from datetime import datetime, timezone, timedelta from typing import Union +import json # Explicit adapter for datetime -> ISO string, replacing the deprecated default # behaviour removed in Python 3.12+. Compatible with Python 3.8+. @@ -14,7 +15,7 @@ class DatabaseProcessor(AbstractDatabaseProcessor): - def __init__(self, database_path: Path): + def __init__(self, database_path: Path, log_removed=None): """This function should handle the connection to the database And if required the creation of the tables""" self.database_path = database_path @@ -22,6 +23,8 @@ def __init__(self, database_path: Path): path = Path(self.database_path) path.parent.mkdir(exist_ok=True, parents=True) self.connection: sqlite3.Connection + self.log_removed_path = log_removed.path if log_removed else None + self.log_removed_types = log_removed.types if log_removed else [] # create tables if required self.open_database() self._create_tables() @@ -384,6 +387,40 @@ def _get_run_paths(self): run_paths[entry["run_start"]] = entry.get("path") or "" return run_paths + def _get_run_data(self, run_start): + cursor = self.connection.cursor() + cursor.execute(GET_RUN_INFO_BY_RUN_START, (run_start,)) + row = cursor.fetchone() + if not row: + return None + columns = [col[0] for col in cursor.description] + return dict(zip(columns, row)) + + def _get_rows_for_run_start(self, table, run_start): + cursor = self.connection.cursor() + cursor.execute(f"SELECT * FROM {table} WHERE run_start = ?", (run_start,)) + rows = cursor.fetchall() + columns = [col[0] for col in cursor.description] + return [dict(zip(columns, row)) for row in rows] + + def _collect_log_entry(self, run_start): + entry = {} + types = self.log_removed_types + include_all = "all" in types + if include_all or "run" in types: + entry["run"] = self._get_run_data(run_start) + if include_all or "suite" in types: + entry["suites"] = self._get_rows_for_run_start("suites", run_start) + if include_all or "test" in types: + entry["tests"] = self._get_rows_for_run_start("tests", run_start) + if include_all or "keyword" in types: + entry["keywords"] = self._get_rows_for_run_start("keywords", run_start) + return entry + + def _log_run_jsonl(self, logpath, entry): + with Path(logpath).open("a", encoding="utf-8") as f: + _ = f.write(json.dumps(entry, default=str) + "\n") + def remove_runs(self, remove_runs: list): """This function removes all provided runs and all their corresponding data""" run_starts, run_names, run_aliases, run_tags, _ = self._get_runs() @@ -526,13 +563,18 @@ def _remove_by_age(self, run_query: str, run_starts: list): def _remove_run(self, run_start: str): """Helper function to remove the data from all tables""" - self.connection.cursor().execute(DELETE_FROM_RUNS.format(run_start=run_start)) - self.connection.cursor().execute(DELETE_FROM_SUITES.format(run_start=run_start)) - self.connection.cursor().execute(DELETE_FROM_TESTS.format(run_start=run_start)) - self.connection.cursor().execute( - DELETE_FROM_KEYWORDS.format(run_start=run_start) - ) - self.connection.commit() + entry = self._collect_log_entry(run_start) if self.log_removed_path else None + with self.connection: + cursor = self.connection.cursor() + cursor.execute(DELETE_FROM_RUNS.format(run_start=run_start)) + if cursor.rowcount > 0: + cursor.execute(DELETE_FROM_SUITES.format(run_start=run_start)) + cursor.execute(DELETE_FROM_TESTS.format(run_start=run_start)) + cursor.execute(DELETE_FROM_KEYWORDS.format(run_start=run_start)) + # Log inside the transaction: if the write fails, the transaction + # rolls back and the run is not deleted. + if self.log_removed_path and entry: + self._log_run_jsonl(self.log_removed_path, entry) def vacuum_database(self): """This function vacuums the database to reduce the size after removing runs""" diff --git a/robotframework_dashboard/main.py b/robotframework_dashboard/main.py index 91152b25..b4fbec95 100644 --- a/robotframework_dashboard/main.py +++ b/robotframework_dashboard/main.py @@ -40,6 +40,7 @@ def main(): arguments.timezone, arguments.log_url, arguments.custom_filters, + arguments.log_removed, ) # If arguments.start_server is provided override some required args if arguments.start_server: diff --git a/robotframework_dashboard/queries.py b/robotframework_dashboard/queries.py index 16d08686..0724292e 100644 --- a/robotframework_dashboard/queries.py +++ b/robotframework_dashboard/queries.py @@ -46,3 +46,5 @@ UPDATE_RUN_PATH = """ UPDATE runs SET path="{path}" WHERE run_start="{run_start}" """ VACUUM_DATABASE = """ VACUUM """ + +GET_RUN_INFO_BY_RUN_START = """ SELECT * FROM runs WHERE run_start = ? """ diff --git a/robotframework_dashboard/robotdashboard.py b/robotframework_dashboard/robotdashboard.py index 5ffab8cf..d17f48c9 100644 --- a/robotframework_dashboard/robotdashboard.py +++ b/robotframework_dashboard/robotdashboard.py @@ -7,6 +7,7 @@ from pathlib import Path from datetime import datetime from typing import Optional +from .arguments import LogRemovedConfig class RobotDashboard: """Class that provides all functionality that robotdashboard has to offer @@ -34,6 +35,7 @@ def __init__( timezone: str = "", log_url: Optional[str] = None, custom_filters: str = "", + log_removed: Optional[LogRemovedConfig] = None, ): """Sets the parameters provided in the command line""" self.database_path = database_path @@ -57,16 +59,23 @@ def __init__( self.timezone = timezone self.log_url = log_url self.custom_filters = custom_filters + self.log_removed = log_removed def initialize_database(self, suppress=True): """Function that initializes the database if it does not exist Also makes a connection that is used internally in the RobotDashboard class functions """ console = "" + if self.log_removed and self.log_removed.path: + log_parent = Path(self.log_removed.path).parent + if not log_parent.exists(): + message = f" ERROR: Directory for '--logremoved' does not exist: '{log_parent}'" + console += self._print_console(message) + raise FileNotFoundError(message) if not suppress: console += self._print_console(f" 1. Database preparation") if not self.database_class: - self.database = DatabaseProcessor(self.database_path) + self.database = DatabaseProcessor(self.database_path, self.log_removed) else: if not suppress: console += self._print_console( @@ -80,6 +89,14 @@ def initialize_database(self, suppress=True): module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) self.database = module.DatabaseProcessor(self.database_path) + is_rm_log_unsupported = not hasattr(self.database, "log_removed_path") + self.database.log_removed_path = self.log_removed.path if self.log_removed else None + self.database.log_removed_types = self.log_removed.types if self.log_removed else [] + if is_rm_log_unsupported and self.log_removed and not suppress: + console += self._print_console( + " WARNING The custom database class does not explicitly support '--logremoved'. " + "Logging might be skipped if the custom class overrides the deletion logic." + ) if not suppress: console += self._print_console( f" created database: '{self.database_path}'" diff --git a/tests/python/test_arguments.py b/tests/python/test_arguments.py index 6aabadd0..c0a677a7 100644 --- a/tests/python/test_arguments.py +++ b/tests/python/test_arguments.py @@ -140,6 +140,7 @@ def _make_namespace(**kwargs): "ssl_keyfile": None, "logurl": None, "custom_filters": None, + "logremoved": None, } defaults.update(kwargs) return argparse.Namespace(**defaults) diff --git a/tests/python/test_database.py b/tests/python/test_database.py index bee7b824..25c75a3d 100644 --- a/tests/python/test_database.py +++ b/tests/python/test_database.py @@ -4,6 +4,7 @@ import pytest from robotframework_dashboard.database import DatabaseProcessor from robotframework_dashboard.processors import OutputProcessor +from robotframework_dashboard.arguments import LogRemovedConfig OUTPUTS_DIR = Path(__file__).parent.parent / "robot" / "resources" / "outputs" SAMPLE_XML = OUTPUTS_DIR / "output-20250313-002134.xml" @@ -506,6 +507,86 @@ def test_get_data_null_test_tags_and_id(db): assert data["tests"][0]["tags"] == "" +# --- _get_run_data --- +def test_get_run_data_returns_dict_for_existing_run(populated_db): + populated_db.open_database() + run_start = populated_db.get_data()["runs"][0]["run_start"] + result = populated_db._get_run_data(run_start) + populated_db.close_database() + assert result is not None + assert result["run_start"] == run_start + + +def test_get_run_data_returns_none_for_missing_run(populated_db): + populated_db.open_database() + result = populated_db._get_run_data("2000-01-01 00:00:00.000000") + populated_db.close_database() + assert result is None + + +# --- _log_run_jsonl --- + +def test_log_run_jsonl_creates_file_and_writes_entry(tmp_path): + import json + db = DatabaseProcessor(tmp_path / "test.db") + logpath = tmp_path / "removed.jsonl" + db._log_run_jsonl(logpath, {"run_start": "2025-01-01", "name": "My Run"}) + parsed = json.loads(logpath.read_text()) + assert parsed["run_start"] == "2025-01-01" + + +def test_log_run_jsonl_appends_on_successive_calls(tmp_path): + db = DatabaseProcessor(tmp_path / "test.db") + logpath = tmp_path / "removed.jsonl" + db._log_run_jsonl(logpath, {"run_start": "2025-01-01"}) + db._log_run_jsonl(logpath, {"run_start": "2025-01-02"}) + assert len(logpath.read_text().splitlines()) == 2 + + +def test_log_run_jsonl_serializes_datetime(tmp_path): + import json + from datetime import datetime + db = DatabaseProcessor(tmp_path / "test.db") + logpath = tmp_path / "removed.jsonl" + db._log_run_jsonl(logpath, {"run_start": datetime(2025, 1, 1, 12, 0, 0)}) + parsed = json.loads(logpath.read_text()) + assert "2025-01-01" in parsed["run_start"] + + +# --- _remove_run with logging --- + +def test_remove_run_with_logging_writes_jsonl_and_deletes(tmp_path): + import json + logpath = tmp_path / "removed.jsonl" + db = DatabaseProcessor(tmp_path / "test.db", log_removed=LogRemovedConfig(types=["all"], path=str(logpath))) + processor = OutputProcessor(SAMPLE_XML) + processor.get_run_start() + db.open_database() + db.insert_output_data(processor.get_output_data(), [], "alias", SAMPLE_XML, None, timezone="+00:00") + run_start = db.get_data()["runs"][0]["run_start"] + db.remove_runs([f"run_start={run_start}"]) + assert len(db.get_data()["runs"]) == 0 + db.close_database() + lines = logpath.read_text().splitlines() + assert len(lines) == 1 + assert json.loads(lines[0])["run"]["run_alias"] == "alias" + + +def test_remove_run_logging_failure_rolls_back_delete(tmp_path): + from unittest.mock import patch + logpath = tmp_path / "removed.jsonl" + db = DatabaseProcessor(tmp_path / "test.db", log_removed=LogRemovedConfig(types=["all"], path=str(logpath))) + processor = OutputProcessor(SAMPLE_XML) + processor.get_run_start() + db.open_database() + db.insert_output_data(processor.get_output_data(), [], "alias", SAMPLE_XML, None, timezone="+00:00") + run_start = db.get_data()["runs"][0]["run_start"] + with patch.object(db, "_log_run_jsonl", side_effect=OSError("disk full")): + db.remove_runs([f"run_start={run_start}"]) + assert len(db.get_data()["runs"]) == 1 + db.close_database() + + # --- remove_runs bare except branch --- def test_remove_runs_exception_branch_logs_error(populated_db): diff --git a/tests/python/test_robotdashboard.py b/tests/python/test_robotdashboard.py index 3616412e..2e716bc4 100644 --- a/tests/python/test_robotdashboard.py +++ b/tests/python/test_robotdashboard.py @@ -1,8 +1,10 @@ import shutil +import pytest from datetime import datetime from pathlib import Path from robotframework_dashboard.robotdashboard import RobotDashboard +from robotframework_dashboard.arguments import LogRemovedConfig OUTPUTS_DIR = Path(__file__).parent.parent / "robot" / "resources" / "outputs" SAMPLE_XML = OUTPUTS_DIR / "output-20250313-002134.xml" @@ -65,6 +67,20 @@ def test_initialize_database_verbose_returns_console(tmp_path): assert "Database preparation" in console or "created database" in console +def test_initialize_database_raises_if_log_dir_missing(tmp_path): + rd = _make_rd(tmp_path, log_removed=LogRemovedConfig(types=["all"], path=str(tmp_path / "nonexistent" / "log.jsonl"))) + with pytest.raises(FileNotFoundError): + rd.initialize_database() + + +def test_initialize_database_valid_log_path_succeeds(tmp_path): + logdir = tmp_path / "logs" + logdir.mkdir() + rd = _make_rd(tmp_path, log_removed=LogRemovedConfig(types=["all"], path=str(logdir / "removed.jsonl"))) + rd.initialize_database() + assert rd.database is not None + + # --- process_outputs --- def test_process_outputs_no_outputs_returns_skip(tmp_path): diff --git a/tests/robot/resources/cli_output/help.txt b/tests/robot/resources/cli_output/help.txt index df425bfb..fa9c3354 100644 --- a/tests/robot/resources/cli_output/help.txt +++ b/tests/robot/resources/cli_output/help.txt @@ -8,12 +8,12 @@ ====================================================================================== usage: robotdashboard [[]-v[]] [[]-h[]] [[]-o [[]PATH ...[]][]] [[]-f [[]PATH ...[]][]] [[]--projectversion VERSION[]] [[]--customfilters FILTERS[]] - [[]-z OFFSET[]] [[]-d PATH[]] [[]-r [[]QUERY ...[]][]] [[]-c PATH[]] - [[]--novacuum [[]BOOL[]][]] [[]-g [[]BOOL[]][]] [[]-l [[]BOOL[]][]] [[]-n NAME[]] - [[]-t TITLE[]] [[]-q INT[]] [[]--offlinedependencies [[]BOOL[]][]] - [[]-u [[]BOOL[]][]] [[]--logurl URL[]] [[]-j PATH[]] - [[]--forcejsonconfig [[]BOOL[]][]] [[]-m PATH[]] - [[]-s [[]HOST:PORT:USERNAME:PASSWORD[]][]] + [[]-z OFFSET[]] [[]-d PATH[]] [[]-r [[]QUERY ...[]][]] + [[]--logremoved TYPES_PATH[]] [[]-c PATH[]] [[]--novacuum [[]BOOL[]][]] + [[]-g [[]BOOL[]][]] [[]-l [[]BOOL[]][]] [[]-n NAME[]] [[]-t TITLE[]] [[]-q INT[]] + [[]--offlinedependencies [[]BOOL[]][]] [[]-u [[]BOOL[]][]] + [[]--logurl URL[]] [[]-j PATH[]] [[]--forcejsonconfig [[]BOOL[]][]] + [[]-m PATH[]] [[]-s [[]HOST:PORT:USERNAME:PASSWORD[]][]] [[]--noautoupdate [[]BOOL[]][]] [[]--ssl-certfile PATH[]] [[]--ssl-keyfile PATH[]] @@ -72,6 +72,13 @@ database: • '-r age=10d' -> remove runs older than 10 days • '-r age=-10d' -> remove runs younger than 10 days • (y)ear/(d)ay/(h)our/(m)inute/(s)econd supported + --logremoved TYPES_PATH + Log (append if file exists) run data to jsonl before removing from the db + • Supply what types to log (default=all) and (optional) a path to the desired .jsonl + Examples: + • '--logremoved "/tmp/removed_runs.jsonl"' + • '--logremoved "run:suite"' + • '--logremoved "all:/tmp/removed_runs.jsonl"' -c PATH, --databaseclass PATH Path to a custom database class (.py) to override the built-in SQLite engine. • See docs for implementation details