New exceptions: NodeException and StopNodeException

src/__init__.py

@@ -24,6 +24,7 @@
     TimeoutException, \
     CatchUpException, \
     StartNodeException, \
+    StopNodeException, \
     InitNodeException, \
     BackupException, \
     InvalidOperationException
@@ -73,6 +74,7 @@
     "QueryTimeoutException",
     "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException",
     "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat",
+    "StopNodeException",
     "NodeApp",
     "PostgresNode",
     "PortManager",

src/exceptions.py

@@ -160,7 +160,7 @@ def __repr__(self) -> str:
         return result
 
 
-class StartNodeException(TestgresException):
+class NodeException(TestgresException):
     _description: typing.Optional[str]
     _files: typing.Optional[typing.Iterable]
 
@@ -224,6 +224,14 @@ def __repr__(self) -> str:
         return result
 
 
+class StartNodeException(NodeException):
+    pass
+
+
+class StopNodeException(NodeException):
+    pass
+
+
 class InitNodeException(TestgresException):
     _message: typing.Optional[str]
 

src/impl/error_utils.py

@@ -0,0 +1,113 @@
+import traceback
+import typing
+import logging
+
+
+class ErrorUtils:
+    @staticmethod
+    def ExceptionToHumanText(exc: BaseException) -> str:
+        assert isinstance(exc, BaseException)
+
+        if __class__.Helper__GetPrevExc(exc) is None:
+            return __class__.Helper__ExceptionToHumanText__Single(exc)
+
+        return __class__.Helper__ExceptionToHumanText__Chain(exc)
+
+    # --------------------------------------------------------------------
+    @staticmethod
+    def Helper__ExceptionToHumanText__Single(exc: BaseException) -> str:
+        assert isinstance(exc, BaseException)
+
+        if isinstance(exc, AssertionError):
+            err_msg = "Exception ({}).".format(type(exc).__name__)
+            assert type(err_msg) is str
+
+            exc_info_lines = traceback.format_exception(exc)
+            assert type(exc_info_lines) is list
+            err_msg2 = "".join(exc_info_lines).strip()
+
+            if err_msg2 != "":
+                err_msg += " " + err_msg2
+
+            assert type(err_msg) is str
+            return err_msg
+
+        err_msg = "Exception ({})".format(type(exc).__name__)
+        assert type(err_msg) is str
+
+        err_msg2 = str(exc).strip()
+
+        if err_msg2 == "":
+            err_msg += "."
+        else:
+            err_msg += ": " + err_msg2
+
+        assert type(err_msg) is str
+        return err_msg
+
+    # --------------------------------------------------------------------
+    @staticmethod
+    def Helper__ExceptionToHumanText__Chain(exc: BaseException) -> str:
+        assert isinstance(exc, BaseException)
+
+        chain: typing.List[BaseException] = []
+        curr = exc
+
+        processed_exc_ids: typing.Set[int] = set()
+
+        while curr is not None:
+            assert isinstance(curr, BaseException)
+            if id(curr) in processed_exc_ids:
+                logging.error("Cycle in exception chain: {}".format(
+                    " --> ".join(
+                        [type(x).__name__ for x in reversed(chain)],
+                    )
+                ))
+                break
+
+            processed_exc_ids.add(id(curr))
+
+            chain.append(curr)
+            curr = __class__.Helper__GetPrevExc(curr)
+            continue
+
+        lines: typing.List[str] = []
+
+        lines.append("It is a chain of exceptions (len: {}):".format(
+            len(chain),
+        ))
+
+        n = 0
+        for e in reversed(chain):
+            n += 1
+            line1 = "---- {}. Exception ({})".format(n, type(e).__name__)
+            line2 = __class__.Helper__GetExcMsg(e)
+
+            if line2 == "":
+                lines.append(line1)
+            else:
+                lines.append(line1 + ":")
+                lines.append(line2)
+            continue
+
+        lines.append("--------")
+
+        return "\n".join(lines)
+
+    # --------------------------------------------------------------------
+    @staticmethod
+    def Helper__GetPrevExc(exc: BaseException) -> typing.Optional[BaseException]:
+        assert isinstance(exc, BaseException)
+        return exc.__cause__ or exc.__context__
+
+    # --------------------------------------------------------------------
+    @staticmethod
+    def Helper__GetExcMsg(exc: BaseException) -> str:
+        assert isinstance(exc, BaseException)
+
+        if isinstance(exc, AssertionError):
+            exc_info_lines = traceback.format_exception(exc)
+            assert type(exc_info_lines) is list
+            return "".join(exc_info_lines).strip()
+
+        return str(exc)

src/node.py

@@ -46,6 +46,7 @@
     QueryException,     \
     QueryTimeoutException, \
     StartNodeException, \
+    StopNodeException, \
     TimeoutException,   \
     InitNodeException,  \
     TestgresException,  \
@@ -55,6 +56,7 @@
 from .port_manager import PortManager
 from .impl.port_manager__this_host import PortManager__ThisHost
 from .impl.port_manager__generic2 import PortManager__Generic2
+from .impl.error_utils import ErrorUtils
 from .impl import internal_utils
 
 from .logger import TestgresLogger
@@ -1183,15 +1185,23 @@ def slow_start(
                 suppress=suppressed_exceptions,
                 max_attempts=max_attempts,
             )
-        except:  # noqa: E722
-            state = self._get_node_state()
-
-            if state.node_status == NodeStatus.Running:
-                assert type(state.pid) is int
+        except BaseException as exc1:
+            internal_utils.send_log_debug(
+                "Failed to wait for node initialization. {}".format(
+                    ErrorUtils.ExceptionToHumanText(exc1)
+                )
+            )
+            try:
                 self.stop()
-            else:
-                self._manually_started_pm_pid = state.pid
-
+            except BaseException as exc2:
+                internal_utils.send_log_debug(
+                    "Failed to stop a node after initialization failure. {}".format(
+                        ErrorUtils.ExceptionToHumanText(exc2)
+                    )
+                )
+                # Try to stabilization our internal state
+                self._manually_started_pm_pid = self._get_node_state().pid
+                raise
             raise
         return
 
@@ -1401,6 +1411,10 @@ def stop(
                 logfile=self.utils_log_file,
             )
             self._manually_started_pm_pid = None
+        except ExecUtilException as e:
+            msg = 'Cannot stop node'
+            files = self._collect_special_files()
+            raise StopNodeException(msg, files) from e
         finally:
             # always stop the reader thread, even if pg_ctl stop failed,
             # so it can't leak into interpreter shutdown.

tests/test_testgres_common.py

@@ -38,6 +38,7 @@
 
 from src import InitNodeException
 from src import StartNodeException
+from src import StopNodeException
 from src import QueryException
 from src import ExecUtilException
 from src import QueryTimeoutException
@@ -524,27 +525,28 @@ def helper_step_of_failure_during_slow_start(
             with pytest.raises(expected_exception=Exception) as x:
                 node.slow_start(max_attempts=10)
 
+            assert x is not None
+            assert isinstance(x.value, Exception)
+
             TestServices.PrintExceptionOK(x.value)
 
-            assert not node.is_started
-            assert node.status() == NodeStatus.Stopped
-            pass
+            if type(x.value) is StartNodeException:
+                assert x.value.description == "Cannot detect postmaster pid."
+                return False
 
-        assert x is not None
-        assert isinstance(x.value, Exception)
-        assert x.value.__context__ is None
+            if type(x.value) is ConnectionResetError:
+                # pg8000
+                assert "Connection reset by peer" in str(x.value)
+                return False
 
-        if type(x.value) is StartNodeException:
-            assert x.value.description == "Cannot detect postmaster pid."
-            return False
+            if type(x.value) is StopNodeException:
+                return True
 
-        if type(x.value) is ConnectionResetError:
-            # pg8000
-            assert "Connection reset by peer" in str(x.value)
-            return False
+            assert not node.is_started
+            assert node.status() == NodeStatus.Stopped
 
-        assert type(x.value) is QueryTimeoutException
-        return True
+            assert type(x.value) is QueryTimeoutException
+            return True
 
     def test_restart(self, node_svc: PostgresNodeService):
         assert isinstance(node_svc, PostgresNodeService)
@@ -643,7 +645,11 @@ def test_double_stop(self, node_svc: PostgresNodeService):
                 node.stop()
 
             assert x is not None
-            assert "Is server running?" in str(x.value)
+            e1 = x.value
+            assert type(e1) is StopNodeException
+            e2 = e1.__cause__
+            assert type(e2) is ExecUtilException
+            assert "Is server running?" in str(e2)
 
             assert not node.is_started
 
@@ -2039,7 +2045,7 @@ def impl__test_pg_ctl_wait_option(
             try:
                 node.stop(wait=False)
                 break
-            except ExecUtilException as e:
+            except StopNodeException as e:
                 # it's ok to get this exception here since node
                 # could be not started yet
                 logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e))

tests/units/exceptions/StopNodeException/test_set001__constructor.py

@@ -0,0 +1,52 @@
+from src.exceptions import StopNodeException
+from src.exceptions import TestgresException as testgres__TestgresException
+
+
+class TestSet001_Constructor:
+    def test_001__default(self):
+        e = StopNodeException()
+        assert type(e) is StopNodeException
+        assert isinstance(e, testgres__TestgresException)
+        assert e.source is None
+        assert e.message == ""
+        assert e.description is None
+        assert e.files is None
+        assert str(e) == ""
+        assert repr(e) == "StopNodeException()"
+        return
+
+    def test_002__message(self):
+        e = StopNodeException(message="abc\n123")
+        assert type(e) is StopNodeException
+        assert isinstance(e, testgres__TestgresException)
+        assert e.source is None
+        assert e.message == "abc\n123"
+        assert e.description == "abc\n123"
+        assert e.files is None
+        assert str(e) == "abc\n123"
+        assert repr(e) == "StopNodeException(message='abc\\n123')"
+        return
+
+    def test_003__files(self):
+        e = StopNodeException(files=[("f\n1", b'line1\nline2')])
+        assert type(e) is StopNodeException
+        assert isinstance(e, testgres__TestgresException)
+        assert e.source is None
+        assert e.message == "f\n1\n----\nb'line1\\nline2'\n"
+        assert e.description is None
+        assert e.files == [("f\n1", b'line1\nline2')]
+        assert str(e) == "f\n1\n----\nb'line1\\nline2'\n"
+        assert repr(e) == "StopNodeException(files=[('f\\n1', b'line1\\nline2')])"
+        return
+
+    def test_004__all(self):
+        e = StopNodeException(message="mmm", files=[("f\n1", b'line1\nline2')])
+        assert type(e) is StopNodeException
+        assert isinstance(e, testgres__TestgresException)
+        assert e.source is None
+        assert e.message == "mmm\nf\n1\n----\nb'line1\\nline2'\n"
+        assert e.description == "mmm"
+        assert e.files == [("f\n1", b'line1\nline2')]
+        assert str(e) == "mmm\nf\n1\n----\nb'line1\\nline2'\n"
+        assert repr(e) == "StopNodeException(message='mmm', files=[('f\\n1', b'line1\\nline2')])"
+        return