parse_encrypt: Don't let e.g. "*.ext" match files in subdirs

This matches the behavior before 3.4.0.

Silent errors from ls-files to avoid warnings about e.g. directories that
aren't readable and also list files that would have been encrypted had they not
been tracked in git (#521).

Fix the patterns written to info/exclude so that they match the same files as
are encrypted (e.g. *.key should only match .key files in the topdir, not in
subdirs).

Author: erijo

test/conftest.py

@@ -609,14 +609,14 @@ def gnupg(tmpdir_factory, runner):
     env["GNUPGHOME"] = home
 
     # this pre-populates std files in the GNUPGHOME
-    runner(["gpg", "-k"], env=env)
+    runner(["gpg", "-k"], env=env, report=False)
 
     def register_gpg_password(password):
         """Publish a new GPG mock password and flush cached passwords"""
         home.join("mock-password").write(password)
-        runner(["gpgconf", "--reload", "gpg-agent"], env=env)
+        runner(["gpgconf", "--reload", "gpg-agent"], env=env, report=False)
 
     yield data(home, register_gpg_password)
 
-    runner(["gpgconf", "--kill", "gpg-agent"], env=env)
-    runner(["gpgconf", "--remove-socketdir", "gpg-agent"], env=env)
+    runner(["gpgconf", "--kill", "gpg-agent"], env=env, report=False)
+    runner(["gpgconf", "--remove-socketdir", "gpg-agent"], env=env, report=False)

test/test_encryption.py

@@ -92,6 +92,7 @@ def encrypt_targets(yadm_cmd, paths):
     paths.work.join("globs dir/globs file2").write("globs file2")
     expected.append("globs dir/globs file2")
     paths.encrypt.write("globs*\n", mode="a")
+    paths.encrypt.write("globs d*/globs*\n", mode="a")
 
     # blank lines
     paths.encrypt.write("\n        \n\t\n", mode="a")
@@ -404,8 +405,8 @@ def test_encrypt_added_to_exclude(runner, yadm_cmd, paths, gnupg):
 
     run = runner(yadm_cmd("encrypt"), env=env)
 
-    assert "test-encrypt-data" in paths.repo.join("info/exclude").read()
-    assert "original-data" in paths.repo.join("info/exclude").read()
+    assert "test-encrypt-data" in exclude_file.read()
+    assert "original-data" in exclude_file.read()
     assert run.success
     assert run.err == ""
 

test/test_unit_exclude_encrypted.py

@@ -24,7 +24,7 @@ def test_exclude_encrypted(runner, tmpdir, yadm, encrypt_exists, auto_exclude, e
     if exclude == "outdated":
         exclude_file.write(f"original-exclude\n{header}outdated\n", ensure=True)
     elif exclude == "up-to-date":
-        exclude_file.write(f"original-exclude\n{header}test-encrypt-data\n", ensure=True)
+        exclude_file.write(f"original-exclude\n{header}/test-encrypt-data\n", ensure=True)
 
     script = f"""
         YADM_TEST=1 source {yadm}
@@ -42,9 +42,9 @@ def test_exclude_encrypted(runner, tmpdir, yadm, encrypt_exists, auto_exclude, e
         if encrypt_exists:
             assert exclude_file.exists()
             if exclude == "missing":
-                assert exclude_file.read() == f"{header}test-encrypt-data\n"
+                assert exclude_file.read() == f"{header}/test-encrypt-data\n"
             else:
-                assert exclude_file.read() == ("original-exclude\n" f"{header}test-encrypt-data\n")
+                assert exclude_file.read() == ("original-exclude\n" f"{header}/test-encrypt-data\n")
             if exclude != "up-to-date":
                 assert f"Updating {exclude_file}" in run.out
             else:

test/test_unit_parse_encrypt.py

@@ -100,10 +100,11 @@ def create_test_encrypt_data(paths):
     edata += "*card1\n"  # matches same file as the one above
     paths.work.join("wildcard1").write("", ensure=True)
     paths.work.join("wildcard2").write("", ensure=True)
+    paths.work.join("subdir/wildcard1").write("", ensure=True)
     expected.add("wildcard1")
     expected.add("wildcard2")
 
-    edata += "dirwild*\n"
+    edata += "dirwild*/file*\n"
     paths.work.join("dirwildcard/file1").write("", ensure=True)
     paths.work.join("dirwildcard/file2").write("", ensure=True)
     expected.add("dirwildcard/file1")
@@ -125,6 +126,9 @@ def create_test_encrypt_data(paths):
     expected.add("ex ex/file4")
     expected.add("ex ex/file6.text")
 
+    paths.work.join("dirwildcard/file7.ex").write("", ensure=True)
+    expected.add("dirwildcard/file7.ex")
+
     # double star
     edata += "doublestar/**/file*\n"
     edata += "!**/file3\n"

yadm

@@ -61,6 +61,7 @@ PROC_VERSION="/proc/version"
 OPERATING_SYSTEM="Unknown"
 
 ENCRYPT_INCLUDE_FILES="unparsed"
+NO_ENCRYPT_TRACKED_FILES=()
 
 LEGACY_WARNING_ISSUED=0
 INVALID_ALT=()
@@ -1042,6 +1043,12 @@ function encrypt() {
   printf '%s\n' "${ENCRYPT_INCLUDE_FILES[@]}"
   echo
 
+  if [ ${#NO_ENCRYPT_TRACKED_FILES[@]} -gt 0 ]; then
+    echo "Warning: The following files are tracked and will NOT be encrypted:"
+    printf '%s\n' "${NO_ENCRYPT_TRACKED_FILES[@]}"
+    echo
+  fi
+
   # encrypt all files which match the globs
   if tar -f - -c "${ENCRYPT_INCLUDE_FILES[@]}" | _encrypt_to "$YADM_ARCHIVE"; then
     echo "Wrote new file: $YADM_ARCHIVE"
@@ -1468,38 +1475,59 @@ function version() {
 
 function exclude_encrypted() {
 
+  local auto_exclude
   auto_exclude=$(config --bool yadm.auto-exclude)
   [ "$auto_exclude" == "false" ] && return 0
 
-  exclude_path="${YADM_REPO}/info/exclude"
-  newline=$'\n'
-  exclude_flag="# yadm-auto-excludes"
-  exclude_header="${exclude_flag}${newline}"
+  # do nothing if there is no YADM_ENCRYPT
+  [ -e "$YADM_ENCRYPT" ] || return 0
+
+  readonly exclude_path="${YADM_REPO}/info/exclude"
+  readonly newline=$'\n'
+  readonly exclude_flag="# yadm-auto-excludes"
+
+  local exclude_header="${exclude_flag}${newline}"
   exclude_header="${exclude_header}# This section is managed by yadm."
   exclude_header="${exclude_header}${newline}"
   exclude_header="${exclude_header}# Any edits below will be lost."
   exclude_header="${exclude_header}${newline}"
 
-  # do nothing if there is no YADM_ENCRYPT
-  [ -e "$YADM_ENCRYPT" ] || return 0
-
   # read encrypt
-  encrypt_data=""
-  while IFS='' read -r line || [ -n "$line" ]; do
-    encrypt_data="${encrypt_data}${line}${newline}"
+  local encrypt_data=""
+  local pattern
+  while IFS='' read -r pattern || [ -n "$pattern" ]; do
+    case ${pattern:0:1} in
+      \#)
+        pattern=""
+        ;;
+      !)
+        # Prepend / to the pattern so that it matches the same files as in
+        # parse_encrypt (i.e. only from the root)
+        pattern="!/${pattern:1}$newline"
+        ;;
+      *)
+        if ! [[ $pattern =~ ^[[:blank:]]*(#|$) ]]; then
+          pattern="/$pattern$newline"
+        else
+          pattern=""
+        fi
+        ;;
+    esac
+    encrypt_data="${encrypt_data}${pattern}"
   done <"$YADM_ENCRYPT"
 
   # read info/exclude
-  unmanaged=""
-  managed=""
+  local unmanaged=""
+  local managed=""
   if [ -e "$exclude_path" ]; then
-    flag_seen=0
+    local -i flag_seen=0
+    local line
     while IFS='' read -r line || [ -n "$line" ]; do
       [ "$line" = "$exclude_flag" ] && flag_seen=1
-      if [ "$flag_seen" -eq 0 ]; then
-        unmanaged="${unmanaged}${line}${newline}"
-      else
+      if ((flag_seen)); then
         managed="${managed}${line}${newline}"
+      else
+        unmanaged="${unmanaged}${line}${newline}"
       fi
     done <"$exclude_path"
   fi
@@ -1926,26 +1954,44 @@ function parse_encrypt() {
   local -a exclude
   local -a include
 
-  while IFS= read -r pattern; do
-    case $pattern in
-      \#*)
+  local pattern
+  while IFS='' read -r pattern || [ -n "$pattern" ]; do
+    case ${pattern:0:1} in
+      \#)
         # Ignore comments
         ;;
-      !*)
-        exclude+=("--exclude=${pattern:1}")
+      !)
+        exclude+=("--exclude=/${pattern:1}")
         ;;
       *)
-        if ! [[ $pattern =~ ^[[:blank:]]*$ ]]; then
+        if ! [[ $pattern =~ ^[[:blank:]]*(#|$) ]]; then
           include+=("$pattern")
         fi
         ;;
     esac
   done <"$YADM_ENCRYPT"
 
-  if [[ ${#include} -gt 0 ]]; then
-    while IFS= read -r filename; do
-      ENCRYPT_INCLUDE_FILES+=("${filename%/}")
-    done <<<"$("$GIT_PROGRAM" ls-files --others "${exclude[@]}" -- "${include[@]}")"
+  if [ ${#include[@]} -gt 0 ]; then
+    while IFS='' read -r filename; do
+      if [ -n "$filename" ]; then
+        ENCRYPT_INCLUDE_FILES+=("${filename%/}")
+      fi
+    done <<<"$(
+      "$GIT_PROGRAM" --glob-pathspecs ls-files --others \
+        "${exclude[@]}" -- "${include[@]}" 2>/dev/null
+    )"
+
+    [ "$YADM_COMMAND" = "encrypt" ] || return
+
+    # List files that matches encryption pattern but is tracked
+    while IFS='' read -r filename; do
+      if [ -n "$filename" ]; then
+        NO_ENCRYPT_TRACKED_FILES+=("${filename%/}")
+      fi
+    done <<<"$(
+      "$GIT_PROGRAM" --glob-pathspecs ls-files \
+        "${exclude[@]}" -- "${include[@]}"
+    )"
   fi
 }