PostgresNode::slow_start and tests are updated (#313)

PostgresNode::slow_start stops a node if it fails to wait for finish of initialization. Client expects to have fully started node.

TestTestgresLocal::test_port_conflict is updated. Checks of node statuses are added.

TestTestgresCommon::test_double_start is updated. Checks of node statuses are added.

TestTestgresCommon::test_uninitialized_start is updated. Checks of node statuses are added.

TestTestgresCommon::test_restart is updated. Checks of node statuses are added.

TestTestgresCommon::test_the_same_port is updated. Normalization. Checks of node statuses are added.

TestTestgresCommon::test_node_app__make_empty_with_explicit_port is rewritten. It does 5 attempts to start a node with a manually defined port number.

Author: dmitry-lipetsk

src/node.py

@@ -975,19 +975,31 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem
 
         self.start(exec_env=exec_env)
 
-        if replica:
-            query = 'SELECT pg_is_in_recovery()'
-        else:
-            query = 'SELECT not pg_is_in_recovery()'
-        # Call poll_query_until until the expected value is returned
-        self.poll_query_until(query=query,
-                              dbname=dbname,
-                              username=username or self.os_ops.username,
-                              suppress={InternalError,
-                                        QueryException,
-                                        ProgrammingError,
-                                        OperationalError},
-                              max_attempts=max_attempts)
+        try:
+            if replica:
+                query = 'SELECT pg_is_in_recovery()'
+            else:
+                query = 'SELECT not pg_is_in_recovery()'
+
+            # Call poll_query_until until the expected value is returned
+            suppressed_exceptions = {
+                InternalError,
+                QueryException,
+                ProgrammingError,
+                OperationalError
+            }
+
+            self.poll_query_until(
+                query=query,
+                dbname=dbname,
+                username=username or self.os_ops.username,
+                suppress=suppressed_exceptions,
+                max_attempts=max_attempts,
+            )
+        except:  # noqa: E722
+            self.stop()
+            raise
+        return
 
     def start(self, params=[], wait=True, exec_env=None):
         """

tests/test_testgres_common.py

@@ -210,8 +210,10 @@ def test_double_start(self, node_svc: PostgresNodeService):
         with __class__.helper__get_node(node_svc) as node:
             node.init()
             assert not node.is_started
+            assert node.status() == NodeStatus.Stopped
             node.start()
             assert node.is_started
+            assert node.status() == NodeStatus.Running
 
             with pytest.raises(expected_exception=StartNodeException) as x:
                 # can't start node more than once
@@ -226,6 +228,7 @@ def test_double_start(self, node_svc: PostgresNodeService):
             assert x.value.message.startswith(x.value.description)
 
             assert node.is_started
+            assert node.status() == NodeStatus.Running
 
         return
 
@@ -272,9 +275,14 @@ def test_uninitialized_start(self, node_svc: PostgresNodeService):
 
         with __class__.helper__get_node(node_svc) as node:
             # node is not initialized yet
+            assert node.status() == NodeStatus.Uninitialized
+
             with pytest.raises(expected_exception=StartNodeException):
                 node.start()
 
+            assert node.status() == NodeStatus.Uninitialized
+        return
+
     def test_restart(self, node_svc: PostgresNodeService):
         assert isinstance(node_svc, PostgresNodeService)
 
@@ -288,11 +296,16 @@ def test_restart(self, node_svc: PostgresNodeService):
             res = node.execute('select 2')
             assert (res == [(2,)])
 
+            assert node.status() == NodeStatus.Running
+
             # restart, fail
             with pytest.raises(expected_exception=StartNodeException):
                 node.append_conf('pg_hba.conf', 'DUMMY')
                 node.restart()
 
+            assert node.status() == NodeStatus.Stopped
+        return
+
     def test_double_stop(self, node_svc: PostgresNodeService):
         assert isinstance(node_svc, PostgresNodeService)
 
@@ -1618,13 +1631,18 @@ def test_the_same_port(self, node_svc: PostgresNodeService):
             with __class__.helper__get_node(node_svc, port=node.port) as node2:
                 assert (type(node2.port) == int)  # noqa: E721
                 assert (node2.port == node.port)
-                assert not (node2._should_free_port)
+                assert (not node2._should_free_port)
+                assert (node2.status() == NodeStatus.Uninitialized)
+
+                node2.init()
 
                 with pytest.raises(
                     expected_exception=StartNodeException,
                     match=re.escape("Cannot start node")
                 ):
-                    node2.init().start()
+                    node2.start()
+
+                assert (node2.status() == NodeStatus.Stopped)
 
             # node is still working
             assert (node.port == node_port_copy)
@@ -2124,6 +2142,8 @@ def test_node_app__make_empty_with_explicit_port(self, node_svc: PostgresNodeSer
         assert node_svc.port_manager is not None
         assert isinstance(node_svc.port_manager, PortManager)
 
+        C_MAX_ATTEMPTS = 5
+
         tmp_dir = node_svc.os_ops.mkdtemp()
         assert tmp_dir is not None
         assert type(tmp_dir) == str  # noqa: E721
@@ -2141,37 +2161,83 @@ def test_node_app__make_empty_with_explicit_port(self, node_svc: PostgresNodeSer
         assert type(node_app.nodes_to_cleanup) == list  # noqa: E721
         assert len(node_app.nodes_to_cleanup) == 0
 
-        port = node_app.port_manager.reserve_port()
-        assert type(port) == int  # noqa: E721
-
-        node: PostgresNode = None
+        attempt = 0
+        ports = []
         try:
-            node = node_app.make_simple("node", port=port)
-            assert node is not None
-            assert isinstance(node, PostgresNode)
-            assert node.os_ops is node_svc.os_ops
-            assert node.port_manager is None  # <---------
-            assert node.port == port
-            assert node._should_free_port == False  # noqa: E712
+            while True:
+                if attempt == C_MAX_ATTEMPTS:
+                    raise RuntimeError("Node did not start.")
 
-            assert type(node_app.nodes_to_cleanup) == list  # noqa: E721
-            assert len(node_app.nodes_to_cleanup) == 1
-            assert node_app.nodes_to_cleanup[0] is node
+                attempt += 1
 
-            node.slow_start()
-        finally:
-            if node is not None:
-                node.stop()
-                node.free_port()
+                logging.info("------------- attempt #{}".format(
+                    attempt
+                ))
 
-        assert node._port is None
-        assert not node._should_free_port
+                port = node_app.port_manager.reserve_port()
+                assert type(port) == int  # noqa: E721
+                assert port is not ports
 
-        node.cleanup(release_resources=True)
+                try:
+                    ports.append(port)
+                except:  # noqa: E722
+                    node_app.port_manager.release_port(port)
+                    raise
+
+                assert len(ports) == attempt
+                node_name = "node_{}".format(attempt)
+
+                logging.info("Node [{}] is creating...".format(node_name))
+                node = node_app.make_simple(node_name, port=port)
+                assert node is not None
+                assert isinstance(node, PostgresNode)
+                assert node.os_ops is node_svc.os_ops
+                assert node.port_manager is None  # <---------
+                assert node.port == port
+                assert node._should_free_port == False  # noqa: E712
+
+                assert type(node_app.nodes_to_cleanup) == list  # noqa: E721
+                assert len(node_app.nodes_to_cleanup) == attempt
+                assert node_app.nodes_to_cleanup[-1] is node
+
+                assert node.status() == NodeStatus.Stopped
+                logging.info("Node is created")
+
+                logging.info("Try to start a node...")
+                try:
+                    node.slow_start()
+                except StartNodeException as e:
+                    logging.info("Exception ({}): {}".format(
+                        type(e).__name__,
+                        e
+                    ))
+                    assert node.status() == NodeStatus.Stopped
+                    node_app.port_manager.release_port(port)
+                    continue
+
+                assert node.status() == NodeStatus.Running
+                logging.info("Node is started")
+
+                logging.info("Stop node")
+                node.stop()
+                assert node.status() == NodeStatus.Stopped
+                logging.info("Node is stopped")
+
+                logging.info("OK. Go home.")
+                assert node is not None
+                assert isinstance(node, PostgresNode)
+                assert node._port is not None
+                assert node._port == port
+                assert not node._should_free_port
+                break
+        finally:
+            while len(ports) > 0:
+                node_app.port_manager.release_port(ports.pop())
 
         # -----------
         logging.info("temp directory [{}] is deleting".format(tmp_dir))
-        node_svc.os_ops.rmdir(tmp_dir)
+        node_svc.os_ops.rmdirs(tmp_dir)
+        return
 
     @staticmethod
     def helper__get_node(

tests/test_testgres_local.py

@@ -12,6 +12,7 @@
 from src import StartNodeException
 from src import ExecUtilException
 from src import NodeApp
+from src import NodeStatus
 from src import scoped_config
 from src import get_new_node
 from src import get_bin_path
@@ -316,7 +317,7 @@ def test_port_conflict(self):
                     assert (node2.port == node1.port)
 
                     node2.init()
-
+                    assert (node2.status() == NodeStatus.Stopped)
                     with pytest.raises(
                         expected_exception=StartNodeException,
                         match=re.escape("Cannot start node after multiple attempts.")
@@ -327,14 +328,16 @@ def test_port_conflict(self):
                     assert (node2._should_free_port)
                     assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 1)
                     assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE)
-                    assert not (node2.is_started)
+                    assert (not node2.is_started)
+                    assert (node2.status() == NodeStatus.Stopped)
 
                 # node2 must release our dummyPort (node1.port)
                 assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0)
 
             # node1 is still working
             assert (node1.port == node1_port_copy)
             assert (node1._should_free_port)
+            assert (node1.status() == NodeStatus.Running)
             assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n')
 
     def test_simple_with_bin_dir(self):