Merge pull request #2552 from Accenture/copilot/fix-shared-de-warning

fix/1834 build does not find shared DEs when run on child BU

Author: JoernBerkefeld

@types/lib/metadataTypes/DataExtension.d.ts

@@ -1004,12 +1004,12 @@ class Mcdev {
                     `Found ${additionalAssetDependencies.length - initialAssetNumber} additional assets linked via ContentBlockByX.`
                 );
             }
-            // reset cache in case this is used progammatically somehow
-            Asset.getJsonFromFSCache = null;
 
             // remove duplicates in main object after adding dependencies
             typeKeyList[depType] = [...new Set(typeKeyList[depType])];
         }
+        // reset cache regardless of whether assets were found, to avoid stale data in programmatic use
+        MetadataTypeInfo['asset'].getJsonFromFSCache = null;
 
         return typeKeyList;
     }

@types/lib/metadataTypes/DataExtension.d.ts.map

@@ -19,6 +19,7 @@ import { checkbox } from '@inquirer/prompts';
  * @typedef {import('../../types/mcdev.d.js').MetadataTypeMap} MetadataTypeMap
  * @typedef {import('../../types/mcdev.d.js').SoapRequestParams} SoapRequestParams
  * @typedef {import('../../types/mcdev.d.js').TemplateMap} TemplateMap
+ * @typedef {import('../../types/mcdev.d.js').TypeKeyCombo} TypeKeyCombo
  */
 
 /**
@@ -1664,6 +1665,63 @@ class DataExtension extends MetadataType {
             return super.getFilesToCommit(keyArr);
         }
     }
+    /**
+     * Hook called when dependency keys were not found in the child BU retrieve folder.
+     * Checks the `_ParentBU_` folder for shared or synchronized dataExtensions and emits
+     * readable warnings. Does NOT add the found items to the deployment package.
+     * Only active when running on a child BU (eid !== mid).
+     * Used by {@link MetadataType.getDependentFiles}.
+     *
+     * @param {string[]} notFound keys that were not found in the child BU retrieve folder
+     * @returns {Promise.<string[]>} keys that should still trigger the default "not found" warning
+     */
+    static async handleNotFoundDependencies(notFound) {
+        // on parent BU, shared DE check is not applicable
+        if (this.buObject.eid === this.buObject.mid) {
+            return notFound;
+        }
+        const parentBUPath = File.normalizePath([
+            this.properties.directories.retrieve,
+            this.buObject.credential,
+            Util.parentBuName,
+            this.definition.type,
+        ]);
+        const stillNotFound = [];
+        for (const key of notFound) {
+            const filePath = File.normalizePath([
+                parentBUPath,
+                File.filterIllegalFilenames(key) + '.' + this.definition.type + '-meta.json',
+            ]);
+            let metadataItem;
+            try {
+                metadataItem = await File.readJson(filePath);
+            } catch {
+                stillNotFound.push(key);
+                continue;
+            }
+            const contentType = metadataItem.r__folder_ContentType;
+            if (contentType === 'shared_dataextension') {
+                Util.logger.warn(
+                    `dataExtension '${key}' is a shared dataExtension stored in _ParentBU_. It cannot be included in the child BU deployment package.`
+                );
+                Util.logger.warn(
+                    `References to it may need to be changed during deployment via mcdev templating or manually. To update the shared dataExtension itself, run build for the parent BU and deploy it separately.`
+                );
+            } else if (contentType === 'synchronizeddataextension') {
+                Util.logger.warn(
+                    `dataExtension '${key}' is a synchronized dataExtension stored in _ParentBU_. It cannot be included in the child BU deployment package.`
+                );
+                Util.logger.warn(
+                    `The reference to it likely needs to be changed during deployment via mcdev templating or manually.`
+                );
+            } else {
+                // regular dataExtension found on parent BU cannot be used from a child BU; treat as not found
+                stillNotFound.push(key);
+            }
+        }
+        return stillNotFound;
+    }
+
     /**
      * helper for {@link MetadataType.createOrUpdate}
      *

@types/lib/metadataTypes/MetadataType.d.ts

@@ -2785,14 +2785,17 @@ class MetadataType {
             (key) => !multiTypeKeyList[this.definition.type].includes(key)
         );
         if (notFound && notFound.length) {
-            Util.logger.warn(
-                Util.getGrayMsg(
-                    `    ☇ skipping ${this.definition.type} dependenc${notFound.length === 1 ? 'y' : 'ies'} ${notFound.join(', ')}: Not found on your in your project folder.`
-                )
-            );
-            // make sure we don't search for it twice
-            notFoundList[this.definition.type] ||= [];
-            notFoundList[this.definition.type].push(...notFound);
+            const remainingNotFound = await this.handleNotFoundDependencies(notFound);
+            if (remainingNotFound.length) {
+                Util.logger.warn(
+                    Util.getGrayMsg(
+                        `    ☇ skipping ${this.definition.type} dependenc${remainingNotFound.length === 1 ? 'y' : 'ies'} ${remainingNotFound.join(', ')}: Not found on your in your project folder.`
+                    )
+                );
+                // make sure we don't search for it twice
+                notFoundList[this.definition.type] ||= [];
+                notFoundList[this.definition.type].push(...remainingNotFound);
+            }
         }
         return multiTypeKeyList;
     }
@@ -2805,6 +2808,19 @@ class MetadataType {
      */
     static getDependentFilesExtra(metadataItem, dependentTypeKeyCombo) {}
 
+    /**
+     * Hook called when dependency keys were not found in the primary retrieve folder.
+     * Override in subtypes to show type-specific warnings (e.g. shared/synchronized dataExtensions).
+     * Keys returned by this method still receive the generic "not found" warning.
+     * Used by {@link MetadataType.getDependentFiles}.
+     *
+     * @param {string[]} notFound keys that could not be found in the retrieve folder
+     * @returns {Promise.<string[]>} keys that should still trigger the default "not found" warning
+     */
+    static async handleNotFoundDependencies(notFound) {
+        return notFound;
+    }
+
     /**
      * helper for {@link MetadataType.getDependentFiles}
      *

@types/lib/metadataTypes/MetadataType.d.ts.map

@@ -0,0 +1,53 @@
+{
+    "CustomerKey": "{{{prefix}}}dataExtensionShared",
+    "DataRetentionPeriodLength": 6,
+    "Description": "{{{description}}}",
+    "Fields": [
+        {
+            "DefaultValue": "",
+            "FieldType": "Text",
+            "IsPrimaryKey": false,
+            "IsRequired": false,
+            "MaxLength": 50,
+            "Name": "FirstName"
+        },
+        {
+            "DefaultValue": "",
+            "FieldType": "Text",
+            "IsPrimaryKey": false,
+            "IsRequired": false,
+            "MaxLength": 50,
+            "Name": "LastName"
+        },
+        {
+            "DefaultValue": "",
+            "FieldType": "EmailAddress",
+            "IsPrimaryKey": false,
+            "IsRequired": true,
+            "MaxLength": 254,
+            "Name": "EmailAddress"
+        },
+        {
+            "DefaultValue": "",
+            "FieldType": "Text",
+            "IsPrimaryKey": true,
+            "IsRequired": true,
+            "MaxLength": 50,
+            "Name": "ContactKey"
+        }
+    ],
+    "IsSendable": true,
+    "IsTestable": true,
+    "Name": "{{{prefix}}}dataExtensionShared",
+    "ResetRetentionPeriodOnImport": false,
+    "SendableDataExtensionField": {
+        "Name": "ContactKey"
+    },
+    "SendableSubscriberField": {
+        "Name": "Subscriber Key"
+    },
+    "c__dataRetentionPeriodUnitOfMeasure": "Months",
+    "c__retentionPolicy": "individialRecords",
+    "r__folder_ContentType": "shared_dataextension",
+    "r__folder_Path": "Shared Items/Shared Data Extensions"
+}

lib/index.js

@@ -0,0 +1,8 @@
+{
+    "name": "{{{prefix}}}query_SharedDE",
+    "key": "{{{prefix}}}query_SharedDE",
+    "description": "{{{description}}}",
+    "r__dataExtension_key": "{{{prefix}}}dataExtensionShared",
+    "targetUpdateTypeName": "Overwrite",
+    "r__folder_Path": "Query"
+}

lib/metadataTypes/DataExtension.js

@@ -0,0 +1,7 @@
+SELECT
+    SubscriberKey AS testField,
+    TRIM(last_name) AS name
+FROM
+    {{{prefix}}}dataExtensionShared
+WHERE
+    country IN ({{{countryCodeIn}}})