patch 9.1.0820: tests: Mac OS tests are too flaky

Problem:  tests: Mac OS tests are too flaky
Solution: Increase max test timeout to 25 minutes,
          allow up to 10 retries on Mac OS runners,
          refactor runtest.vim (Milly).

closes: #15940

Co-authored-by: K.Takata <kentkt@csc.jp>
Signed-off-by: Milly <milly.ca@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>

Author: 2 people

.github/workflows/ci.yml

@@ -389,7 +389,7 @@ jobs:
           brew install diffutils
 
       - name: Test
-        timeout-minutes: 20
+        timeout-minutes: 25
         run: |
           make ${TEST}
 

src/testdir/runtest.vim

@@ -190,10 +190,6 @@ function GetAllocId(name)
   return lnum - top - 1
 endfunc
 
-if has('reltime')
-  let g:func_start = reltime()
-endif
-
 " Get the list of swap files in the current directory.
 func s:GetSwapFileList()
   let save_dir = &directory
@@ -603,6 +599,16 @@ for g:testfunc in sort(s:tests)
   " A test can set g:test_is_flaky to retry running the test.
   let g:test_is_flaky = 0
 
+  " A test can set g:max_run_nr to change the max retry count.
+  let g:max_run_nr = 5
+  if has('mac')
+    let g:max_run_nr = 10
+  endif
+
+  " By default, give up if the same error occurs.  A test can set
+  " g:giveup_same_error to 0 to not give up on the same error and keep trying.
+  let g:giveup_same_error = 1
+
   let starttime = strftime("%H:%M:%S")
   call RunTheTest(g:testfunc)
 
@@ -618,10 +624,15 @@ for g:testfunc in sort(s:tests)
       call extend(s:messages, v:errors)
 
       let endtime = strftime("%H:%M:%S")
-      call add(total_errors, $'Run {g:run_nr}, {starttime} - {endtime}:')
+      if has('reltime')
+        let suffix = $' in{reltimestr(reltime(g:func_start))} seconds'
+      else
+        let suffix = ''
+      endif
+      call add(total_errors, $'Run {g:run_nr}, {starttime} - {endtime}{suffix}:')
       call extend(total_errors, v:errors)
 
-      if g:run_nr >= 5 || prev_error == v:errors[0]
+      if g:run_nr >= g:max_run_nr || g:giveup_same_error && prev_error == v:errors[0]
         call add(total_errors, 'Flaky test failed too often, giving up')
         let v:errors = total_errors
         break
@@ -632,7 +643,8 @@ for g:testfunc in sort(s:tests)
       " Flakiness is often caused by the system being very busy.  Sleep a
       " couple of seconds to have a higher chance of succeeding the second
       " time.
-      sleep 2
+      let delay = g:run_nr * 2
+      exe 'sleep' delay
 
       let prev_error = v:errors[0]
       let v:errors = []

src/testdir/screendump.vim

@@ -56,6 +56,7 @@ func VerifyScreenDump(buf, filename, options, ...)
 
   " Starting a terminal to make a screendump is always considered flaky.
   let g:test_is_flaky = 1
+  let g:giveup_same_error = 0
 
   " wait for the pending updates to be handled.
   call TermWait(a:buf)
@@ -83,41 +84,55 @@ func VerifyScreenDump(buf, filename, options, ...)
     sleep 50m
     call delete(testfile)
     call term_dumpwrite(a:buf, testfile, a:options)
+
+    if refdump->empty()
+      let msg = 'See new dump file: call term_dumpload("testdir/' .. testfile .. '")'
+      call assert_report(msg)
+      " no point in retrying
+      let g:run_nr = 10
+      return 1
+    endif
+
     let testdump = ReadAndFilter(testfile, filter)
     if refdump == testdump
       call delete(testfile)
       if did_mkdir
 	call delete('failed', 'd')
       endif
+      if i > 0
+	call remove(v:errors, -1)
+      endif
       break
     endif
-    if i == max_loops
-      " Leave the failed dump around for inspection.
-      if filereadable(reference)
-	let msg = 'See dump file difference: call term_dumpdiff("testdir/' .. testfile .. '", "testdir/' .. reference .. '")'
-	if a:0 == 1
-	  let msg = a:1 . ': ' . msg
-	endif
-	if len(testdump) != len(refdump)
-	  let msg = msg . '; line count is ' . len(testdump) . ' instead of ' . len(refdump)
-	endif
-      else
-	let msg = 'See new dump file: call term_dumpload("testdir/' .. testfile .. '")'
-	" no point in retrying
-	let g:run_nr = 10
+
+    " Leave the failed dump around for inspection.
+    let msg = 'See dump file difference: call term_dumpdiff("testdir/' .. testfile .. '", "testdir/' .. reference .. '")'
+    if a:0 == 1
+      let msg = a:1 . ': ' . msg
+    endif
+    if len(testdump) != len(refdump)
+      let msg = msg . '; line count is ' . len(testdump) . ' instead of ' . len(refdump)
+    endif
+    for j in range(len(refdump))
+      if j >= len(testdump)
+	break
       endif
-      for i in range(len(refdump))
-	if i >= len(testdump)
-	  break
-	endif
-	if testdump[i] != refdump[i]
-	  let msg = msg . '; difference in line ' . (i + 1) . ': "' . testdump[i] . '"'
-	endif
-      endfor
-      call assert_report(msg)
-      return 1
+      if testdump[j] != refdump[j]
+	let msg = msg . '; difference in line ' . (j + 1) . ': "' . testdump[j] . '"'
+      endif
+    endfor
+
+    " Always add the last error so that it is displayed on timeout.
+    " See TestTimeout() in runtest.vim.
+    if i > 0
+      call remove(v:errors, -1)
     endif
+    call assert_report(msg)
+
     let i += 1
+    if i >= max_loops
+      return 1
+    endif
   endwhile
   return 0
 endfunc

src/testdir/test_channel.vim

@@ -2762,6 +2762,8 @@ func LspTests(port)
 endfunc
 
 func Test_channel_lsp_mode()
+  " The channel lsp mode test is flaky and gives the same error.
+  let g:giveup_same_error = 0
   call RunServer('test_channel_lsp.py', 'LspTests', [])
 endfunc
 

src/version.c

@@ -704,6 +704,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    820,
 /**/
     819,
 /**/