Merge pull request #2456 from Accenture/develop

8.3.1

Author: JoernBerkefeld

@types/lib/metadataTypes/DataFilter.d.ts

@@ -20,6 +20,8 @@ import File from '../util/file.js';
  * @typedef {import('../../types/mcdev.d.js').SoapRequestParams} SoapRequestParams
  * @typedef {import('../../types/mcdev.d.js').TemplateMap} TemplateMap
  * @typedef {import('../../types/mcdev.d.js').DataFilterItem} DataFilterItem
+ * @typedef {import('../../types/mcdev.d.js').DataExtensionFieldMap} DataExtensionFieldMap
+ * @typedef {import('../../types/mcdev.d.js').DataExtensionFieldItem} DataExtensionFieldItem
  * @typedef {import('../../types/mcdev.d.js').DataFilterMap} DataFilterMap
  * @typedef {import('../../types/mcdev.d.js').MultiMetadataTypeMap} MultiMetadataTypeMap
  * @typedef {import('../../types/mcdev.d.js').FilterConditionSet} FilterConditionSet
@@ -201,12 +203,18 @@ class DataFilter extends MetadataType {
                 : Object.values(metadataTypeMap)
                       .filter((item) => item.r__source_dataExtension_key)
                       .map((item) => item.r__source_dataExtension_key)
+                      .filter(
+                          (deKey) =>
+                              !this.cache.dataExtensionField ||
+                              !this.cache.dataExtensionField[deKey]
+                      )
                       .filter(Boolean);
         if (deKeys.length) {
+            const deKeysUnique = [...new Set(deKeys)];
             Util.logger.info(' - Caching dependent Metadata: dataExtensionField');
             // only proceed with the download if we have dataExtension keys
             const fieldOptions = {};
-            for (const deKey of deKeys) {
+            for (const deKey of deKeysUnique) {
                 fieldOptions.filter = fieldOptions.filter
                     ? {
                           leftOperand: {
@@ -226,9 +234,32 @@ class DataFilter extends MetadataType {
             DataExtensionField.buObject = this.buObject;
             DataExtensionField.client = this.client;
             DataExtensionField.properties = this.properties;
-            this.dataExtensionFieldCache = (
-                await DataExtensionField.retrieveForCacheDE(fieldOptions, ['Name', 'ObjectID'])
-            ).metadata;
+            this.saveDataExtensionFieldCacheToMap(
+                (await DataExtensionField.retrieveForCacheDE(fieldOptions, ['Name', 'ObjectID']))
+                    .metadata
+            );
+        }
+    }
+    /**
+     * helper for {@link DataFilter._cacheDeFields}
+     *
+     * @param {DataExtensionFieldMap} deFieldCache -
+     */
+    static saveDataExtensionFieldCacheToMap(deFieldCache) {
+        this.cache.dataExtensionField ||= {};
+
+        if (!deFieldCache) {
+            return;
+        }
+        for (const field of Object.values(deFieldCache)) {
+            if (!field?.DataExtension?.CustomerKey) {
+                continue;
+            }
+            if (!this.cache.dataExtensionField[field.DataExtension.CustomerKey]) {
+                /** @type {Object.<string, DataExtensionFieldItem[]>} */
+                this.cache.dataExtensionField[field.DataExtension.CustomerKey] = [];
+            }
+            this.cache.dataExtensionField[field.DataExtension.CustomerKey].push(field);
         }
     }
     /**
@@ -399,7 +430,7 @@ class DataFilter extends MetadataType {
      *
      * @param {DataFilterItem} metadata -
      * @param {'postRetrieve'|'preDeploy'} mode -
-     * @param {object[]} [fieldCache] -
+     * @param {DataExtensionFieldItem[]} [fieldCache] -
      * @param {FilterConditionSet} [filter] -
      * @returns {void}
      */
@@ -408,7 +439,7 @@ class DataFilter extends MetadataType {
             return this._resolveFields(
                 metadata,
                 mode,
-                Object.values(this.dataExtensionFieldCache),
+                this.cache.dataExtensionField[metadata.r__source_dataExtension_key],
                 metadata.c__filterDefinition?.ConditionSet
             );
         }
@@ -418,11 +449,11 @@ class DataFilter extends MetadataType {
                 : [filter.Condition];
             if (mode === 'postRetrieve') {
                 for (const condition of conditionsArr) {
-                    this._postRetrieve_resolveFieldIdsCondition(condition, fieldCache);
+                    this._postRetrieve_resolveFieldIdsCondition(metadata, condition, fieldCache);
                 }
             } else if (mode === 'preDeploy') {
                 for (const condition of conditionsArr) {
-                    this._preDeploy_resolveFieldNamesCondition(condition, fieldCache);
+                    this._preDeploy_resolveFieldNamesCondition(metadata, condition, fieldCache);
                 }
             }
         }
@@ -438,16 +469,21 @@ class DataFilter extends MetadataType {
     /**
      * helper for {@link _resolveFields}
      *
+     * @param {DataFilterItem} metadata -
      * @param {FilterCondition} condition -
-     * @param {object[]} fieldCache -
+     * @param {DataExtensionFieldItem[]} fieldCache -
      * @returns {void}
      */
-    static _postRetrieve_resolveFieldIdsCondition(condition, fieldCache) {
+    static _postRetrieve_resolveFieldIdsCondition(metadata, condition, fieldCache) {
         condition.r__dataExtensionField_name = fieldCache.find(
             (field) => field.ObjectID === condition['@_ID']
         )?.Name;
         if (condition.r__dataExtensionField_name) {
             delete condition['@_ID'];
+        } else {
+            Util.logger.warn(
+                ` - ${this.definition.type} ${metadata[this.definition.keyField]}: could not resolve dataExtensionField with ID ${condition['@_ID']} in condition`
+            );
         }
         if (['IsEmpty', 'IsNotEmpty'].includes(condition['@_Operator'])) {
             delete condition.Value;
@@ -456,16 +492,21 @@ class DataFilter extends MetadataType {
     /**
      * helper for {@link _resolveFields}
      *
+     * @param {DataFilterItem} metadata -
      * @param {FilterCondition} condition -
-     * @param {object[]} fieldCache -
+     * @param {DataExtensionFieldItem[]} fieldCache -
      * @returns {void}
      */
-    static _preDeploy_resolveFieldNamesCondition(condition, fieldCache) {
+    static _preDeploy_resolveFieldNamesCondition(metadata, condition, fieldCache) {
         condition['@_ID'] = fieldCache.find(
             (field) => field.Name === condition.r__dataExtensionField_name
         )?.ObjectID;
         if (condition['@_ID']) {
             delete condition.r__dataExtensionField_name;
+        } else {
+            throw new Error(
+                ` - ${this.definition.type} ${metadata[this.definition.keyField]}: could not resolve dataExtensionField with Name ${condition.r__dataExtensionField_name} in condition`
+            );
         }
         if (['IsEmpty', 'IsNotEmpty'].includes(condition['@_Operator'])) {
             condition.Value ||= '';

@types/lib/metadataTypes/DataFilter.d.ts.map

@@ -753,6 +753,19 @@ class MetadataType {
         return false;
     }
 
+    /**
+     * helper for {@link MetadataType.upsert} to enforce max key length
+     *
+     * @param {string} metadataKey key of metadata
+     */
+    static enforceMaxKeyLength(metadataKey) {
+        if (this.definition.maxKeyLength && metadataKey.length > this.definition.maxKeyLength) {
+            throw new Error(
+                `key exceeds maximum length of ${this.definition.maxKeyLength} characters for this type`
+            );
+        }
+    }
+
     /**
      * MetadataType upsert, after retrieving from target and comparing to check if create or update operation is needed.
      *
@@ -774,6 +787,7 @@ class MetadataType {
                 // preDeployTasks parsing
                 let deployableMetadata;
                 try {
+                    this.enforceMaxKeyLength(metadataKey);
                     metadataMap[metadataKey] = await this.validation(
                         'deploy',
                         metadataMap[metadataKey],

@types/lib/metadataTypes/MetadataType.d.ts

@@ -1,5 +1,5 @@
 {
-    "customerKey": "testExisting_asset_htmlblock-matchName-fail",
+    "customerKey": "testExisting_asset_html-matchNamFail",
     "assetType": {
         "name": "htmlblock",
         "displayName": "HTML Block"

@types/lib/metadataTypes/MetadataType.d.ts.map

@@ -1,5 +1,5 @@
 {
-    "customerKey": "testExisting_asset_htmlblock-matchName",
+    "customerKey": "testExisting_asset_html-matchName",
     "assetType": {
         "name": "htmlblock",
         "displayName": "HTML Block"