Merge pull request #206 from MarketSquare/parse-script-improvements

Fix #184 Parse script improvements

Author: amochin

src/DatabaseLibrary/query.py

@@ -13,6 +13,7 @@
 #  limitations under the License.
 
 import inspect
+import re
 import sys
 from typing import List, Optional
 
@@ -208,11 +209,17 @@ def delete_all_rows_from_table(self, tableName: str, sansTran: bool = False, ali
             if cur and not sansTran:
                 db_connection.client.rollback()
 
-    def execute_sql_script(self, sqlScriptFileName: str, sansTran: bool = False, alias: Optional[str] = None):
+    def execute_sql_script(
+        self, sqlScriptFileName: str, sansTran: bool = False, split: bool = True, alias: Optional[str] = None
+    ):
         """
         Executes the content of the `sqlScriptFileName` as SQL commands. Useful for setting the database to a known
         state before running your tests, or clearing out your test data after running each a test.
 
+        SQL commands are expected to be delimited by a semicolon (';') - they will be split and executed separately.
+        You can disable this behaviour setting the parameter `split` to _False_ -
+        in this case the entire script content will be passed to the database module for execution.
+
         Sample usage :
         | Execute Sql Script | ${EXECDIR}${/}resources${/}DDL-setup.sql |
         | Execute Sql Script | ${EXECDIR}${/}resources${/}DML-setup.sql |
@@ -221,7 +228,6 @@ def execute_sql_script(self, sqlScriptFileName: str, sansTran: bool = False, ali
         | Execute Sql Script | ${EXECDIR}${/}resources${/}DML-teardown.sql |
         | Execute Sql Script | ${EXECDIR}${/}resources${/}DDL-teardown.sql |
 
-        SQL commands are expected to be delimited by a semicolon (';') - they will be executed separately.
 
         For example:
         DELETE FROM person_employee_table;
@@ -272,76 +278,101 @@ def execute_sql_script(self, sqlScriptFileName: str, sansTran: bool = False, ali
         with open(sqlScriptFileName, encoding="UTF-8") as sql_file:
             cur = None
             try:
-                statements_to_execute = []
                 cur = db_connection.client.cursor()
                 logger.info(f"Executing : Execute SQL Script  |  {sqlScriptFileName}")
-                current_statement = ""
-                inside_statements_group = False
-
-                for line in sql_file:
-                    line = line.strip()
-                    if line.startswith("#") or line.startswith("--") or line == "/":
-                        continue
-                    if line.lower().startswith("begin"):
-                        inside_statements_group = True
-
-                    # semicolons inside the line? use them to separate statements
-                    # ... but not if they are inside a begin/end block (aka. statements group)
-                    sqlFragments = line.split(";")
-                    # no semicolons
-                    if len(sqlFragments) == 1:
-                        current_statement += line + " "
-                        continue
-                    quotes = 0
-                    # "select * from person;" -> ["select..", ""]
-                    for sqlFragment in sqlFragments:
-                        if len(sqlFragment.strip()) == 0:
+                if not split:
+                    logger.info("Statements splitting disabled - pass entire script content to the database module")
+                    self.__execute_sql(cur, sql_file.read())
+                else:
+                    logger.info("Splitting script file into statements...")
+                    statements_to_execute = []
+                    current_statement = ""
+                    inside_statements_group = False
+                    proc_start_pattern = re.compile("create( or replace)? (procedure|function){1}( )?")
+                    proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?")
+                    for line in sql_file:
+                        line = line.strip()
+                        if line.startswith("#") or line.startswith("--") or line == "/":
                             continue
-                        if inside_statements_group:
-                            # if statements inside a begin/end block have semicolns,
-                            # they must persist - even with oracle
-                            sqlFragment += "; "
-                        if sqlFragment.lower() == "end; ":
-                            inside_statements_group = False
-                        elif sqlFragment.lower().startswith("begin"):
+
+                        # check if the line matches the creating procedure regexp pattern
+                        if proc_start_pattern.match(line.lower()):
+                            inside_statements_group = True
+                        elif line.lower().startswith("begin"):
                             inside_statements_group = True
 
-                        # check if the semicolon is a part of the value (quoted string)
-                        quotes += sqlFragment.count("'")
-                        quotes -= sqlFragment.count("\\'")
-                        quotes -= sqlFragment.count("''")
-                        inside_quoted_string = quotes % 2 != 0
-                        if inside_quoted_string:
-                            sqlFragment += ";"  # restore the semicolon
-
-                        current_statement += sqlFragment
-                        if not inside_statements_group and not inside_quoted_string:
-                            statements_to_execute.append(current_statement.strip())
-                            current_statement = ""
-                            quotes = 0
-
-                current_statement = current_statement.strip()
-                if len(current_statement) != 0:
-                    statements_to_execute.append(current_statement)
-
-                for statement in statements_to_execute:
-                    logger.info(f"Executing statement from script file: {statement}")
-                    omit_semicolon = not statement.lower().endswith("end;")
-                    self.__execute_sql(cur, statement, omit_semicolon)
+                        # semicolons inside the line? use them to separate statements
+                        # ... but not if they are inside a begin/end block (aka. statements group)
+                        sqlFragments = line.split(";")
+                        # no semicolons
+                        if len(sqlFragments) == 1:
+                            current_statement += line + " "
+                            continue
+                        quotes = 0
+                        # "select * from person;" -> ["select..", ""]
+                        for sqlFragment in sqlFragments:
+                            if len(sqlFragment.strip()) == 0:
+                                continue
+
+                            if inside_statements_group:
+                                # if statements inside a begin/end block have semicolns,
+                                # they must persist - even with oracle
+                                sqlFragment += "; "
+
+                            if proc_end_pattern.match(sqlFragment.lower()):
+                                inside_statements_group = False
+                            elif proc_start_pattern.match(sqlFragment.lower()):
+                                inside_statements_group = True
+                            elif sqlFragment.lower().startswith("begin"):
+                                inside_statements_group = True
+
+                            # check if the semicolon is a part of the value (quoted string)
+                            quotes += sqlFragment.count("'")
+                            quotes -= sqlFragment.count("\\'")
+                            quotes -= sqlFragment.count("''")
+                            inside_quoted_string = quotes % 2 != 0
+                            if inside_quoted_string:
+                                sqlFragment += ";"  # restore the semicolon
+
+                            current_statement += sqlFragment
+                            if not inside_statements_group and not inside_quoted_string:
+                                statements_to_execute.append(current_statement.strip())
+                                current_statement = ""
+                                quotes = 0
+
+                    current_statement = current_statement.strip()
+                    if len(current_statement) != 0:
+                        statements_to_execute.append(current_statement)
+
+                    for statement in statements_to_execute:
+                        logger.info(f"Executing statement from script file: {statement}")
+                        line_ends_with_proc_end = re.compile(r"(\s|;)" + proc_end_pattern.pattern + "$")
+                        omit_semicolon = not line_ends_with_proc_end.search(statement.lower())
+                        self.__execute_sql(cur, statement, omit_semicolon)
                 if not sansTran:
                     db_connection.client.commit()
             finally:
                 if cur and not sansTran:
                     db_connection.client.rollback()
 
     def execute_sql_string(
-        self, sqlString: str, sansTran: bool = False, alias: Optional[str] = None, parameters: Optional[List] = None
+        self,
+        sqlString: str,
+        sansTran: bool = False,
+        omitTrailingSemicolon: Optional[bool] = None,
+        alias: Optional[str] = None,
+        parameters: Optional[List] = None,
     ):
         """
-        Executes the sqlString as SQL commands. Useful to pass arguments to your sql.
-        SQL commands are expected to be delimited by a semicolon (';').
+        Executes the ``sqlString`` as a single SQL command.
 
-        Use optional `sansTran` to run command without an explicit transaction commit or rollback:
+        Use optional ``sansTran`` to run command without an explicit transaction commit or rollback.
+
+        Use optional ``omitTrailingSemicolon`` parameter for explicit instruction,
+        if the trailing semicolon (;) at the SQL string end should be removed or not:
+        - Some database modules (e.g. Oracle) throw an exception, if you leave a semicolon at the string end
+        - However, there are exceptional cases, when you need it even for Oracle - e.g. at the end of a PL/SQL block.
+        - If not specified, it's decided based on the current database module in use. For Oracle, the semicolon is removed by default.
 
         Use optional ``alias`` parameter to specify what connection should be used for the query if you have more
         than one connection open.
@@ -353,6 +384,7 @@ def execute_sql_string(
         | Execute Sql String | DELETE FROM person_employee_table; DELETE FROM person_table |
         | Execute Sql String | DELETE FROM person_employee_table; DELETE FROM person_table | alias=my_alias |
         | Execute Sql String | DELETE FROM person_employee_table; DELETE FROM person_table | sansTran=True |
+        | Execute Sql String | CREATE PROCEDURE proc AS BEGIN DBMS_OUTPUT.PUT_LINE('Hello!'); END; | omitTrailingSemicolon=False |
         | @{parameters} | Create List |  person_employee_table |
         | Execute Sql String | SELECT * FROM %s | parameters=${parameters} |
         """
@@ -361,7 +393,7 @@ def execute_sql_string(
         try:
             cur = db_connection.client.cursor()
             logger.info(f"Executing : Execute SQL String  |  {sqlString}")
-            self.__execute_sql(cur, sqlString, parameters=parameters)
+            self.__execute_sql(cur, sqlString, omit_trailing_semicolon=omitTrailingSemicolon, parameters=parameters)
             if not sansTran:
                 db_connection.client.commit()
         finally:

test/readme.md

@@ -40,7 +40,7 @@ See the folder `.github/workflows`
 
 ## Microsoft SQL Server
 - https://hub.docker.com/_/microsoft-mssql-server
-- docker run --rm --name mssql -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=MyPass1234!" -p 1433:1433 -d mcr.microsoft.com/mssql/server
+- docker run --rm --name mssql -e ACCEPT_EULA=Y -e MSSQL_SA_PASSWORD='MyPass1234!' -p 1433:1433 -d mcr.microsoft.com/mssql/server
 --> login and create DB:
     - docker exec -it mssql bash
     - /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P 'MyPass1234!'

…ources/create_stored_procedure_mssql.sql …urces/create_stored_procedures_mssql.sqltest/resources/create_stored_procedure_mssql.sql renamed to test/resources/create_stored_procedures_mssql.sql test/resources/create_stored_procedure_mssql.sql renamed to test/resources/create_stored_procedures_mssql.sql

@@ -33,4 +33,18 @@ BEGIN
 SELECT FIRST_NAME FROM person;
 SELECT LAST_NAME FROM person;
 RETURN;
+END;
+
+DROP PROCEDURE IF EXISTS check_condition;
+CREATE PROCEDURE check_condition
+AS
+DECLARE @v_condition BIT = 1;
+IF @v_condition = 1
+BEGIN
+PRINT 'Condition is true';
+END
+ELSE
+BEGIN
+PRINT 'Condition is false';
+END
 END;

…ources/create_stored_procedure_mysql.sql …urces/create_stored_procedures_mysql.sqltest/resources/create_stored_procedure_mysql.sql renamed to test/resources/create_stored_procedures_mysql.sql test/resources/create_stored_procedure_mysql.sql renamed to test/resources/create_stored_procedures_mysql.sql

@@ -27,4 +27,15 @@ CREATE PROCEDURE get_all_first_and_second_names()
 BEGIN
 SELECT FIRST_NAME FROM person;
 SELECT LAST_NAME FROM person;
-END;
+END;
+
+DROP PROCEDURE IF EXISTS check_condition;
+CREATE PROCEDURE check_condition()
+BEGIN
+   DECLARE v_condition BOOLEAN DEFAULT TRUE;
+   IF v_condition THEN
+      SELECT 'Condition is true' AS Result;
+   ELSE
+      SELECT 'Condition is false' AS Result;
+   END IF;
+END

test/resources/create_stored_procedures_oracle.sql

@@ -29,4 +29,15 @@ OPEN first_names_cursor for
 SELECT FIRST_NAME FROM person;
 OPEN second_names_cursor for
 SELECT LAST_NAME FROM person;
-END;
+END;
+
+CREATE OR REPLACE PROCEDURE 
+check_condition AS
+v_condition BOOLEAN := TRUE;
+BEGIN
+IF v_condition THEN
+DBMS_OUTPUT.PUT_LINE('Condition is true');
+ELSE
+DBMS_OUTPUT.PUT_LINE('Condition is false');
+END IF;
+END check_condition;

…ces/create_stored_procedure_postgres.sql …es/create_stored_procedures_postgres.sqltest/resources/create_stored_procedure_postgres.sql renamed to test/resources/create_stored_procedures_postgres.sql test/resources/create_stored_procedure_postgres.sql renamed to test/resources/create_stored_procedures_postgres.sql

@@ -49,4 +49,23 @@ RETURN NEXT result1;
 OPEN result2 FOR SELECT LAST_NAME FROM person;
 RETURN NEXT result2;
 END
+';
+
+DROP ROUTINE IF EXISTS check_condition;
+CREATE FUNCTION
+check_condition()
+RETURNS VOID
+LANGUAGE plpgsql
+AS
+'
+DECLARE
+   v_condition BOOLEAN := TRUE;
+   v_res BOOLEAN := TRUE;
+BEGIN
+   IF v_condition THEN
+      v_res := TRUE;
+   ELSE
+      v_res := FALSE;
+   END IF;
+END
 ';

test/tests/common_tests/stored_procedures.robot

@@ -88,18 +88,21 @@ Procedure Returns Multiple Result Sets
     Should Be Equal    ${second result set}[0][0]    See
     Should Be Equal    ${second result set}[1][0]    Schneider
 
+Procedure With IF/ELSE Block
+    Call Stored Procedure    check_condition
+
 
 *** Keywords ***
 Create And Fill Tables And Stored Procedures
     Create Person Table And Insert Data
     IF    "${DB_MODULE}" in ["oracledb", "cx_Oracle"]
         Execute SQL Script    ${CURDIR}/../../resources/create_stored_procedures_oracle.sql
     ELSE IF    "${DB_MODULE}" in ["pymysql"]
-        Execute SQL Script    ${CURDIR}/../../resources/create_stored_procedure_mysql.sql
+        Execute SQL Script    ${CURDIR}/../../resources/create_stored_procedures_mysql.sql
     ELSE IF    "${DB_MODULE}" in ["psycopg2", "psycopg3"]
-        Execute SQL Script    ${CURDIR}/../../resources/create_stored_procedure_postgres.sql
+        Execute SQL Script    ${CURDIR}/../../resources/create_stored_procedures_postgres.sql
     ELSE IF    "${DB_MODULE}" in ["pymssql"]
-        Execute SQL Script    ${CURDIR}/../../resources/create_stored_procedure_mssql.sql
+        Execute SQL Script    ${CURDIR}/../../resources/create_stored_procedures_mssql.sql
     ELSE
         Skip    Don't know how to create stored procedures for '${DB_MODULE}'
     END

test/tests/custom_db_tests/oracle_omit_semicolon.robot

@@ -0,0 +1,27 @@
+*** Settings ***
+Documentation    Tests for the parameter _omitTrailingSemicolon_ in the keyword
+...    _Execute SQL String_ - special for the issue #184:
+...    https://github.com/MarketSquare/Robotframework-Database-Library/issues/184
+...    The _PLSQL BLOCK_ is most likely valid for Oracle DB only.
+
+Resource            ../../resources/common.resource
+Suite Setup         Connect To DB
+Suite Teardown      Disconnect From Database
+Test Setup          Create Person Table And Insert Data
+Test Teardown       Drop Tables Person And Foobar
+
+
+*** Variables ***
+${NORMAL QUERY}    SELECT * FROM person;
+${PLSQL BLOCK}    DECLARE ERRCODE NUMBER; ERRMSG VARCHAR2(200); BEGIN DBMS_OUTPUT.PUT_LINE('Hello!'); END;
+
+
+*** Test Cases ***
+Explicitely Omit Semicolon
+    [Documentation]    Check if it works for Oracle - explicitely omitting the semicolon
+    ...    is equal to the default behaviour, otherwise oracle_db throws an error
+    Execute Sql String    ${NORMAL QUERY}    omitTrailingSemicolon=True
+
+Explicitely Dont't Omit Semicolon
+    [Documentation]    Check if it works for Oracle - it throws an error without a semicolon
+    Execute Sql String    ${PLSQL BLOCK}    omitTrailingSemicolon=False

test/tests/custom_db_tests/sql_script_split_commands.robot

@@ -0,0 +1,22 @@
+*** Settings ***
+Documentation    Tests for the parameter _split_ in the keyword
+...    _Execute SQL Script_ - special for the issue #184:
+...    https://github.com/MarketSquare/Robotframework-Database-Library/issues/184
+
+Resource            ../../resources/common.resource
+Suite Setup         Connect To DB
+Suite Teardown      Disconnect From Database
+Test Setup          Create Person Table
+Test Teardown       Drop Tables Person And Foobar
+
+
+*** Test Cases ***
+Split Commands
+    [Documentation]    Such a simple script works always,
+    ...    just check if the logs if the parameter value was processed properly
+    Execute Sql Script    ${CURDIR}/../../resources/insert_data_in_person_table.sql    split=True
+
+Don't Split Commands
+    [Documentation]    Running such a script as a single statement works for PostgreSQL,
+    ...    but fails in Oracle. Check in the logs if the splitting was disabled.
+    Execute Sql Script    ${CURDIR}/../../resources/insert_data_in_person_table.sql    split=False