Refactoring and bug fixing

Dockerfile--altlinux_10.tmpl

@@ -129,11 +129,6 @@ if [ ! -f /home/test/.ssh/id_rsa ]; then \
     chmod 600 /home/test/.ssh/authorized_keys; \
 fi; \
 ls -la /home/test/.ssh/; \
-su -c \"ssh-keyscan -H localhost >> /home/test/.ssh/known_hosts\" test; \
-su -c \"ssh-keyscan -H 127.0.0.1 >> /home/test/.ssh/known_hosts\" test; \
-if [ -n \"${TEST_CFG__REMOTE_HOST:-}\" ]; then \
-    su -c \"ssh-keyscan -H ${TEST_CFG__REMOTE_HOST} >> /home/test/.ssh/known_hosts\" test; \
-fi; \
 if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \
     cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \
     export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \

Dockerfile--altlinux_11.tmpl

@@ -130,11 +130,6 @@ if [ ! -f /home/test/.ssh/id_rsa ]; then \
     chmod 600 /home/test/.ssh/authorized_keys; \
 fi; \
 ls -la /home/test/.ssh/; \
-su -c \"ssh-keyscan -H localhost >> /home/test/.ssh/known_hosts\" test; \
-su -c \"ssh-keyscan -H 127.0.0.1 >> /home/test/.ssh/known_hosts\" test; \
-if [ -n \"${TEST_CFG__REMOTE_HOST:-}\" ]; then \
-    su -c \"ssh-keyscan -H ${TEST_CFG__REMOTE_HOST} >> /home/test/.ssh/known_hosts\" test; \
-fi; \
 if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \
     cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \
     export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \

src/impl/platforms/linux/internal_platform_utils.py

@@ -2,6 +2,7 @@
 
 from .. import internal_platform_utils as base
 from ... import internal_utils
+from ....raise_error import RaiseError
 
 from testgres.operations.os_ops import OsOperations
 from testgres.operations.exceptions import ExecUtilException
@@ -417,21 +418,12 @@ def _find_postmaster__throw_error__fail(
         assert type(data_dir) is str
         assert type(failures) is list
 
-        err_msg = "FindPostmater did {} attempts and has not gotten a stable result. List of failures:\n"
-
-        n = 0
-        sep = ""
-        for e in failures:
-            assert isinstance(e, Exception)
-
-            n += 1
-            err_msg += sep
-            err_msg += "Failure #{}. Exception ({}):\n{}".format(
-                n,
-                type(e).__name__,
-                str(e),
-            )
-            sep = "\n"
-            continue
+        method_name = "InternalPlatformUtils::FindPostmaster(bin_dir={!r}, data_dir={!r})".format(
+            bin_dir,
+            data_dir,
+        )
 
-        raise RuntimeError(err_msg)
+        RaiseError.function_did_multiple_attempts_without_stable_result(
+            method_name,
+            failures,
+        )

src/node.py

@@ -78,6 +78,7 @@
 from .port_manager import PortManager
 from .impl.port_manager__this_host import PortManager__ThisHost
 from .impl.port_manager__generic2 import PortManager__Generic2
+from .impl import internal_utils
 
 from .logger import TestgresLogger
 
@@ -158,6 +159,8 @@ class PostgresNode(object):
     # a max number of node start attempts
     _C_MAX_START_ATEMPTS = 5
 
+    _C_MAX_GET_CHILDREN_ATTEMPTS = 5
+
     _C_PM_PID__IS_NOT_DETECTED = -1
 
     _name: typing.Optional[str]
@@ -448,21 +451,76 @@ def child_processes(self) -> typing.List[ProcessProxy]:
         if x.pid is None:
             assert x.node_status != NodeStatus.Running
             RaiseError.node_err__cant_enumerate_child_processes(
-                x.node_status
+                x.node_status,
             )
 
-        assert x.node_status == NodeStatus.Running
+        assert x.node_status != NodeStatus.Stopped
         assert type(x.pid) is int
         return self._get_child_processes(x.pid)
 
     def _get_child_processes(self, pid: int) -> typing.List[ProcessProxy]:
         assert type(pid) is int
         assert isinstance(self._os_ops, OsOperations)
 
-        # get a list of postmaster's children
-        children = self._os_ops.get_process_children(pid)
+        C_MAX_ATTEMPT_COUNT = __class__. _C_MAX_GET_CHILDREN_ATTEMPTS
+        assert type(C_MAX_ATTEMPT_COUNT) is int
+        assert C_MAX_ATTEMPT_COUNT > 0
+
+        failures: typing.List[Exception] = []
+
+        nAttempt = 0
+
+        while True:
+            assert nAttempt < C_MAX_ATTEMPT_COUNT
 
-        return [ProcessProxy(p) for p in children]
+            nAttempt += 1
+
+            # get a list of postmaster's children
+            children = self._os_ops.get_process_children(pid)
+            assert type(children) is list
+
+            result: typing.List[ProcessProxy] = []
+
+            for p in children:
+                assert hasattr(p, "pid")
+                try:
+                    proxy = ProcessProxy(p)  # raise
+                except Exception as e:
+                    internal_utils.send_log_debug(
+                        "Failed to process a node child process [pid: {}]. Exception ({}): {}".format(
+                            p.pid,
+                            type(e).__name__,
+                            e,
+                        )
+                    )
+                    failures.append(e)
+                    break
+
+                assert type(proxy) is ProcessProxy
+                result.append(proxy)
+                continue
+
+            if len(result) == len(children):
+                return result
+
+            assert len(result) < len(children)
+
+            if nAttempt < C_MAX_ATTEMPT_COUNT:
+                time.sleep(0.05)
+                continue
+            break
+
+        assert nAttempt == C_MAX_ATTEMPT_COUNT
+        assert len(failures) == C_MAX_ATTEMPT_COUNT
+
+        method_name = "PostgresNode::_get_child_processes(pid={!r})".format(
+            pid,
+        )
+
+        RaiseError.function_did_multiple_attempts_without_stable_result(
+            method_name,
+            failures,
+        )
 
     @property
     def source_walsender(self):

src/raise_error.py

@@ -6,14 +6,14 @@
 
 class RaiseError:
     @staticmethod
-    def pg_ctl_returns_an_empty_string(_params):
+    def pg_ctl_returns_an_empty_string(_params) -> typing.NoReturn:
         errLines = []
         errLines.append("Utility pg_ctl returns an empty string.")
         errLines.append("Command line is {0}".format(_params))
         raise RuntimeError("\n".join(errLines))
 
     @staticmethod
-    def pg_ctl_returns_an_unexpected_string(out, _params):
+    def pg_ctl_returns_an_unexpected_string(out, _params) -> typing.NoReturn:
         errLines = []
         errLines.append("Utility pg_ctl returns an unexpected string:")
         errLines.append(out)
@@ -22,7 +22,7 @@ def pg_ctl_returns_an_unexpected_string(out, _params):
         raise RuntimeError("\n".join(errLines))
 
     @staticmethod
-    def pg_ctl_returns_a_zero_pid(out, _params):
+    def pg_ctl_returns_a_zero_pid(out, _params) -> typing.NoReturn:
         errLines = []
         errLines.append("Utility pg_ctl returns a zero pid. Output string is:")
         errLines.append(out)
@@ -33,7 +33,7 @@ def pg_ctl_returns_a_zero_pid(out, _params):
     @staticmethod
     def node_err__cant_enumerate_child_processes(
         node_status: NodeStatus
-    ):
+    ) -> typing.NoReturn:
         assert type(node_status) is NodeStatus
 
         msg = "Can't enumerate node child processes. {}.".format(
@@ -48,7 +48,7 @@ def node_err__cant_enumerate_child_processes(
     @staticmethod
     def node_err__cant_kill(
         node_status: NodeStatus
-    ):
+    ) -> typing.NoReturn:
         assert type(node_status) is NodeStatus
 
         msg = "Can't kill server process. {}.".format(
@@ -60,6 +60,38 @@ def node_err__cant_kill(
 
         raise InvalidOperationException(msg)
 
+    @staticmethod
+    def function_did_multiple_attempts_without_stable_result(
+        function_name: str,
+        failures: typing.List[Exception],
+    ) -> typing.NoReturn:
+        assert type(function_name) is str
+        assert type(failures) is list
+
+        err_msg = "{} did {} attempts and has not gotten a stable result.".format(
+            function_name,
+            len(failures),
+        )
+
+        err_msg += " List of failures:\n"
+
+        n = 0
+        sep = ""
+        for e in failures:
+            assert isinstance(e, Exception)
+
+            n += 1
+            err_msg += sep
+            err_msg += "Failure #{}. Exception ({}):\n{}".format(
+                n,
+                type(e).__name__,
+                str(e),
+            )
+            sep = "\n"
+            continue
+
+        raise InvalidOperationException(err_msg)
+
     @staticmethod
     def _map_node_status_to_reason(
         node_status: NodeStatus,