diff --git a/Dockerfile--altlinux_10.tmpl b/Dockerfile--altlinux_10.tmpl index 5a04aba8..2ec2c7ba 100644 --- a/Dockerfile--altlinux_10.tmpl +++ b/Dockerfile--altlinux_10.tmpl @@ -129,11 +129,6 @@ if [ ! -f /home/test/.ssh/id_rsa ]; then \ chmod 600 /home/test/.ssh/authorized_keys; \ fi; \ ls -la /home/test/.ssh/; \ -su -c \"ssh-keyscan -H localhost >> /home/test/.ssh/known_hosts\" test; \ -su -c \"ssh-keyscan -H 127.0.0.1 >> /home/test/.ssh/known_hosts\" test; \ -if [ -n \"${TEST_CFG__REMOTE_HOST:-}\" ]; then \ - su -c \"ssh-keyscan -H ${TEST_CFG__REMOTE_HOST} >> /home/test/.ssh/known_hosts\" test; \ -fi; \ if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ diff --git a/Dockerfile--altlinux_11.tmpl b/Dockerfile--altlinux_11.tmpl index c46f78bb..f3a3cc9e 100644 --- a/Dockerfile--altlinux_11.tmpl +++ b/Dockerfile--altlinux_11.tmpl @@ -130,11 +130,6 @@ if [ ! -f /home/test/.ssh/id_rsa ]; then \ chmod 600 /home/test/.ssh/authorized_keys; \ fi; \ ls -la /home/test/.ssh/; \ -su -c \"ssh-keyscan -H localhost >> /home/test/.ssh/known_hosts\" test; \ -su -c \"ssh-keyscan -H 127.0.0.1 >> /home/test/.ssh/known_hosts\" test; \ -if [ -n \"${TEST_CFG__REMOTE_HOST:-}\" ]; then \ - su -c \"ssh-keyscan -H ${TEST_CFG__REMOTE_HOST} >> /home/test/.ssh/known_hosts\" test; \ -fi; \ if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ diff --git a/src/impl/platforms/linux/internal_platform_utils.py b/src/impl/platforms/linux/internal_platform_utils.py index 3834f6e7..95d9d15a 100644 --- a/src/impl/platforms/linux/internal_platform_utils.py +++ b/src/impl/platforms/linux/internal_platform_utils.py @@ -2,6 +2,7 @@ from .. import internal_platform_utils as base from ... import internal_utils +from ....raise_error import RaiseError from testgres.operations.os_ops import OsOperations from testgres.operations.exceptions import ExecUtilException @@ -417,21 +418,12 @@ def _find_postmaster__throw_error__fail( assert type(data_dir) is str assert type(failures) is list - err_msg = "FindPostmater did {} attempts and has not gotten a stable result. List of failures:\n" - - n = 0 - sep = "" - for e in failures: - assert isinstance(e, Exception) - - n += 1 - err_msg += sep - err_msg += "Failure #{}. Exception ({}):\n{}".format( - n, - type(e).__name__, - str(e), - ) - sep = "\n" - continue + method_name = "InternalPlatformUtils::FindPostmaster(bin_dir={!r}, data_dir={!r})".format( + bin_dir, + data_dir, + ) - raise RuntimeError(err_msg) + RaiseError.function_did_multiple_attempts_without_stable_result( + method_name, + failures, + ) diff --git a/src/node.py b/src/node.py index be1c04f4..94a3316a 100644 --- a/src/node.py +++ b/src/node.py @@ -78,6 +78,7 @@ from .port_manager import PortManager from .impl.port_manager__this_host import PortManager__ThisHost from .impl.port_manager__generic2 import PortManager__Generic2 +from .impl import internal_utils from .logger import TestgresLogger @@ -158,6 +159,8 @@ class PostgresNode(object): # a max number of node start attempts _C_MAX_START_ATEMPTS = 5 + _C_MAX_GET_CHILDREN_ATTEMPTS = 5 + _C_PM_PID__IS_NOT_DETECTED = -1 _name: typing.Optional[str] @@ -448,10 +451,10 @@ def child_processes(self) -> typing.List[ProcessProxy]: if x.pid is None: assert x.node_status != NodeStatus.Running RaiseError.node_err__cant_enumerate_child_processes( - x.node_status + x.node_status, ) - assert x.node_status == NodeStatus.Running + assert x.node_status != NodeStatus.Stopped assert type(x.pid) is int return self._get_child_processes(x.pid) @@ -459,10 +462,65 @@ def _get_child_processes(self, pid: int) -> typing.List[ProcessProxy]: assert type(pid) is int assert isinstance(self._os_ops, OsOperations) - # get a list of postmaster's children - children = self._os_ops.get_process_children(pid) + C_MAX_ATTEMPT_COUNT = __class__. _C_MAX_GET_CHILDREN_ATTEMPTS + assert type(C_MAX_ATTEMPT_COUNT) is int + assert C_MAX_ATTEMPT_COUNT > 0 + + failures: typing.List[Exception] = [] + + nAttempt = 0 + + while True: + assert nAttempt < C_MAX_ATTEMPT_COUNT - return [ProcessProxy(p) for p in children] + nAttempt += 1 + + # get a list of postmaster's children + children = self._os_ops.get_process_children(pid) + assert type(children) is list + + result: typing.List[ProcessProxy] = [] + + for p in children: + assert hasattr(p, "pid") + try: + proxy = ProcessProxy(p) # raise + except Exception as e: + internal_utils.send_log_debug( + "Failed to process a node child process [pid: {}]. Exception ({}): {}".format( + p.pid, + type(e).__name__, + e, + ) + ) + failures.append(e) + break + + assert type(proxy) is ProcessProxy + result.append(proxy) + continue + + if len(result) == len(children): + return result + + assert len(result) < len(children) + + if nAttempt < C_MAX_ATTEMPT_COUNT: + time.sleep(0.05) + continue + break + + assert nAttempt == C_MAX_ATTEMPT_COUNT + assert len(failures) == C_MAX_ATTEMPT_COUNT + + method_name = "PostgresNode::_get_child_processes(pid={!r})".format( + pid, + ) + + RaiseError.function_did_multiple_attempts_without_stable_result( + method_name, + failures, + ) @property def source_walsender(self): diff --git a/src/raise_error.py b/src/raise_error.py index 61c5e43a..e30c9315 100644 --- a/src/raise_error.py +++ b/src/raise_error.py @@ -6,14 +6,14 @@ class RaiseError: @staticmethod - def pg_ctl_returns_an_empty_string(_params): + def pg_ctl_returns_an_empty_string(_params) -> typing.NoReturn: errLines = [] errLines.append("Utility pg_ctl returns an empty string.") errLines.append("Command line is {0}".format(_params)) raise RuntimeError("\n".join(errLines)) @staticmethod - def pg_ctl_returns_an_unexpected_string(out, _params): + def pg_ctl_returns_an_unexpected_string(out, _params) -> typing.NoReturn: errLines = [] errLines.append("Utility pg_ctl returns an unexpected string:") errLines.append(out) @@ -22,7 +22,7 @@ def pg_ctl_returns_an_unexpected_string(out, _params): raise RuntimeError("\n".join(errLines)) @staticmethod - def pg_ctl_returns_a_zero_pid(out, _params): + def pg_ctl_returns_a_zero_pid(out, _params) -> typing.NoReturn: errLines = [] errLines.append("Utility pg_ctl returns a zero pid. Output string is:") errLines.append(out) @@ -33,7 +33,7 @@ def pg_ctl_returns_a_zero_pid(out, _params): @staticmethod def node_err__cant_enumerate_child_processes( node_status: NodeStatus - ): + ) -> typing.NoReturn: assert type(node_status) is NodeStatus msg = "Can't enumerate node child processes. {}.".format( @@ -48,7 +48,7 @@ def node_err__cant_enumerate_child_processes( @staticmethod def node_err__cant_kill( node_status: NodeStatus - ): + ) -> typing.NoReturn: assert type(node_status) is NodeStatus msg = "Can't kill server process. {}.".format( @@ -60,6 +60,38 @@ def node_err__cant_kill( raise InvalidOperationException(msg) + @staticmethod + def function_did_multiple_attempts_without_stable_result( + function_name: str, + failures: typing.List[Exception], + ) -> typing.NoReturn: + assert type(function_name) is str + assert type(failures) is list + + err_msg = "{} did {} attempts and has not gotten a stable result.".format( + function_name, + len(failures), + ) + + err_msg += " List of failures:\n" + + n = 0 + sep = "" + for e in failures: + assert isinstance(e, Exception) + + n += 1 + err_msg += sep + err_msg += "Failure #{}. Exception ({}):\n{}".format( + n, + type(e).__name__, + str(e), + ) + sep = "\n" + continue + + raise InvalidOperationException(err_msg) + @staticmethod def _map_node_status_to_reason( node_status: NodeStatus,