Added optional handle duplicates step in federated reporting import

This feature removes duplicate hostkey entries during federated reporting
import based on lastreporttimestamp in __hosts table.

Enable this feature by defining a class in augments (def.json):

{
  "classes": {
    "cfengine_mp_fr_handle_duplicate_hostkeys": ["any::"]
  }
}

Ticket: ENT-6035
Changelog: title

Author: craigcomstock

cfe_internal/enterprise/federation/federation.cf

@@ -316,6 +316,8 @@ bundle agent federation_manage_files
     "feeder_username" data => parsejson('{"feeder_username":"$(cfengine_enterprise_federation:config.transport_user)"}');
     "feeder" data => parsejson('{"feeder": "$(sys.key_digest)"}');
     "cf_version" data => parsejson('{"cf_version":"$(sys.cf_version)"}');
+    "handle_duplicates_value" string => ifelse("cfengine_mp_fr_handle_duplicate_hostkeys", "yes", "no");
+    "handle_duplicates" data => parsejson('{"handle_duplicates":"$(handle_duplicates_value)"}');
 
   files:
     enterprise_edition.(policy_server|am_policy_hub)::
@@ -364,7 +366,11 @@ bundle agent federation_manage_files
         create => "true",
         template_method => "mustache",
         edit_template => "$(this.promise_dirname)/../../../templates/federated_reporting/config.sh.mustache",
-        template_data => mergedata(@(login), @(feeder_username), @(feeder), @(cf_version),
+        template_data => mergedata(@(login),
+                                   @(feeder_username),
+                                   @(feeder),
+                                   @(cf_version),
+                                   @(handle_duplicates),
                                    parsejson('{"inventory_refresh_cmd": ""}')),
         perms => default:mog( "640", "root", "$(transport_user)" );
 

templates/federated_reporting/config.sh.mustache

@@ -76,3 +76,5 @@ CFE_FR_IMPORT_FILTERS_DIR="${CFE_FR_IMPORT_FILTERS_DIR:-$DEFAULT_PREFIX/superhub
 CFE_FR_INVENTORY_REFRESH_CMD="{{inventory_refresh_cmd}}"
 CFE_FR_DB_USER="{{db_user}}"
 CFE_FR_DB_USER="${CFE_FR_DB_USER:-cfpostgres}"
+CFE_FR_HANDLE_DUPLICATES="{{handle_duplicates}}"   # default is no (don't handle duplicates as it adds to time to import)
+CFE_FR_HANDLE_DUPLICATES="${CFE_FR_HANDLE_DUPLICATES:-no}"

templates/federated_reporting/import.sh

@@ -17,6 +17,7 @@ true "${CFE_FR_COMPRESSOR_EXT?undefined}"
 true "${CFE_FR_EXTRACTOR?undefined}"
 true "${CFE_FR_TABLES?undefined}"
 true "${CFE_FR_INVENTORY_REFRESH_CMD?undefined}"
+true "${CFE_FR_HANDLE_DUPLICATES?undefined}"
 
 if ! type "$CFE_FR_EXTRACTOR" >/dev/null; then
   log "Extractor $CFE_FR_EXTRACTOR not available!"
@@ -57,8 +58,10 @@ table_whitelist=$(printf "'%s'," $CFE_FR_TABLES | sed -e 's/,$//')
 
 failed=0
 log "Setting up schemas for import"
+declare -a hostkeys
 for file in $dump_files; do
   hostkey=$(basename "$file" | cut -d. -f1)
+  hostkeys+=($hostkey)
   "$CFE_BIN_DIR"/psql -U $CFE_FR_DB_USER -d cfdb --set "ON_ERROR_STOP=1" \
                       -c "SELECT ensure_feeder_schema('$hostkey', ARRAY[$table_whitelist]);" \
     > schema_setup.log 2>&1 || failed=1
@@ -100,6 +103,22 @@ else
   done
 fi
 
+if [ "$CFE_FR_HANDLE_DUPLICATES" = "yes" ]; then
+  log "Handle Duplicate Hostkeys"
+  hostkey_list=$(printf "'%s'," ${hostkeys[*]} | sed -e 's/,$//')
+  "$CFE_BIN_DIR"/psql -U $CFE_FR_DB_USER -d cfdb --set "ON_ERROR_STOP=1" \
+                      -c "SELECT handle_duplicate_hostkeys_in_import(ARRAY[$hostkey_list]);"  \
+    > duplicates.log 2>&1 || failed=1
+  if [ "$failed" = "0" ]; then
+    log "Handle Duplicate Hostkeys: DONE"
+  else
+    log "Handle Duplicate Hostkeys: FAILED"
+    log "last 10 lines of duplicates.log"
+    tail -n 10 duplicates.log
+    exit 1
+  fi
+fi
+
 failed=0
 log "Attaching schemas"
 for file in $dump_files; do