patch 9.2.0245: xxd: color output detection is broken

Problem:  xxd: color output detection is broken
          (Juergen Weigert)
Solution: Fix the issues (Hirohito Higashi)

- Disable auto color when output goes to a file (two-argument form)
- Check TERM variable: disable color when unset, empty or "dumb"
- Add color_forced flag to preserve -R always behavior
- Add tests for the new behavior

fixes:  #19790
closes: #19813

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Signed-off-by: Hirohito Higashi <h.east.727@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>

Author: 2 people

src/testdir/test_xxd.vim

@@ -758,4 +758,68 @@ func Test_xxd_null_dereference()
   bw!
 endfunc
 
+func Test_xxd_color_outfile_no_color()
+  CheckUnix
+
+  " When output goes to a file (two-argument form), auto color should be
+  " disabled and produce the same result as redirect to file.
+  let input = 'Xxd_color_input'
+  let outfile = 'Xxd_color_outfile'
+  let outredir = 'Xxd_color_outredir'
+
+  call writefile([repeat('A', 100)], input)
+
+  " Two-argument form: xxd infile outfile
+  silent exe '!' . s:xxd_cmd . ' ' . input . ' ' . outfile
+
+  " Redirect form: xxd infile > outfile
+  silent exe '!' . s:xxd_cmd . ' ' . input . ' > ' . outredir
+
+  call assert_equal(readfile(outredir), readfile(outfile))
+
+  call delete(input)
+  call delete(outfile)
+  call delete(outredir)
+endfunc
+
+func Test_xxd_color_term_dumb()
+  CheckUnix
+
+  " When TERM is 'dumb', auto color should be disabled.
+  let input = 'Xxd_term_input'
+  let outfile = 'Xxd_term_outfile'
+
+  call writefile([repeat('A', 100)], input)
+
+  silent exe '!' . 'TERM=dumb ' . s:xxd_cmd . ' ' . input . ' > ' . outfile
+  let result = readfile(outfile)
+  " Output should not contain escape sequences
+  for line in result
+    call assert_equal(-1, stridx(line, "\e"), 'Unexpected color escape in: ' . line)
+  endfor
+
+  call delete(input)
+  call delete(outfile)
+endfunc
+
+func Test_xxd_color_term_unset()
+  CheckUnix
+
+  " When TERM is not set, auto color should be disabled.
+  let input = 'Xxd_unset_input'
+  let outfile = 'Xxd_unset_outfile'
+
+  call writefile([repeat('A', 100)], input)
+
+  silent exe '!' . 'env -u TERM ' . s:xxd_cmd . ' ' . input . ' > ' . outfile
+  let result = readfile(outfile)
+  " Output should not contain escape sequences
+  for line in result
+    call assert_equal(-1, stridx(line, "\e"), 'Unexpected color escape in: ' . line)
+  endfor
+
+  call delete(input)
+  call delete(outfile)
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab

src/version.c

@@ -734,6 +734,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    245,
 /**/
     244,
 /**/

src/xxd/xxd.c

@@ -73,6 +73,7 @@
  * 24.08.2025  avoid NULL dereference with autoskip colorless
  * 26.11.2025  update indent in exit_with_usage()
  * 19.03.2026  Add -t option to end output with terminating null
+ * 25.03.2026  Fix color output issues
  *
  * (c) 1990-1998 by Juergen Weigert (jnweiger@gmail.com)
  *
@@ -153,7 +154,7 @@ extern void perror __P((char *));
 # endif
 #endif
 
-char version[] = "xxd 2026-03-19 by Juergen Weigert et al.";
+char version[] = "xxd 2026-03-25 by Juergen Weigert et al.";
 #ifdef WIN32
 char osver[] = " (Win32)";
 #else
@@ -723,7 +724,16 @@ enable_color(void)
   mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
   return (int)SetConsoleMode(out, mode);
 #elif defined(UNIX)
-  return isatty(STDOUT_FILENO);
+  char *term;
+
+  if (!isatty(STDOUT_FILENO))
+    return 0;
+
+  term = getenv("TERM");
+  if (term == NULL || *term == '\0' || !strcmp(term, "dumb"))
+    return 0;
+
+  return 1;
 #else
   return 0;
 #endif
@@ -748,6 +758,7 @@ main(int argc, char *argv[])
   char *varname = NULL;
   int addrlen = 9;
   int color = 0;
+  int color_forced = 0;	/* set when -R always is used */
   char *no_color;
   char cur_color = 0;
 
@@ -920,6 +931,7 @@ main(int argc, char *argv[])
 	    {
 	      (void)enable_color();
 	      color = 1;
+	      color_forced = 1;
 	    }
 	  else if (!STRNCMP(pw, "never", 5))
 	    color = 0;
@@ -1018,6 +1030,10 @@ main(int argc, char *argv[])
 	  return 3;
 	}
       rewind(fpo);
+
+      /* Disable auto color when writing to a file. */
+      if (!color_forced)
+	color = 0;
     }
 
   if (revert)