Add warnings for MVs

Author: Thomas Mahlberg

KustoSchemaTools/Changes/BaseChange.cs

@@ -20,10 +20,10 @@ protected BaseChange(string entityType, string entity, T from, T to)
         public List<DatabaseScriptContainer> Scripts { get; set; } = new List<DatabaseScriptContainer>();
 
 
-        public string Markdown { get; protected set; }
+        public string Markdown { get; set; }
 
         public bool IsAsync { get; set; }
-
+        public Comment Comment { get; set; }
     }
 
 }

KustoSchemaTools/Changes/Comment.cs

@@ -0,0 +1,10 @@
+namespace KustoSchemaTools.Changes
+{
+    public class Comment
+    {
+        public string Text { get; set; }
+        public bool FailsRollout { get; set; }
+        public CommentKind Kind { get; set; }
+
+    }
+}

KustoSchemaTools/Changes/CommentKind.cs

@@ -0,0 +1,11 @@
+namespace KustoSchemaTools.Changes
+{
+    public enum CommentKind
+    {
+        Note,
+        Tip,
+        Important,
+        Warning,
+        Caution
+    }
+}

KustoSchemaTools/Changes/DatabaseChanges.cs

@@ -1,4 +1,5 @@
-using Kusto.Language;
+using Kusto.Cloud.Platform.Utils;
+using Kusto.Language;
 using KustoSchemaTools.Model;
 using Microsoft.Extensions.Logging;
 using System.Data;
@@ -47,7 +48,43 @@ public static List<IChange> GenerateChanges(Database oldState, Database newState
             result.AddRange(GenerateDeletions(oldState, newState.Deletions, log));
 
             result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.Tables, nameof(newState.Tables), log, (oldItem, newItem) => oldItem != null || newItem.Columns?.Any() == true));
-            result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.MaterializedViews, nameof(newState.MaterializedViews), log));
+            var mvChanges = GenerateScriptCompareChanges(oldState, newState, db => db.MaterializedViews, nameof(newState.MaterializedViews), log);
+            foreach(var mvChange in mvChanges)
+            {                                
+                var relevantChange = mvChange.Scripts.FirstOrDefault(itm => itm.Kind== "CreateMaterializedViewAsync");
+                if (relevantChange == null) 
+                    continue;
+
+
+                var newMv = newState.MaterializedViews[mvChange.Entity];
+
+                var specificCache = (newMv.Kind== "table" ?
+                    (newState.Tables.ContainsKey(newMv.Source) ? newState.Tables[newMv.Source].Policies?.HotCache : null) :
+                    (newState.MaterializedViews.ContainsKey(newMv.Source) ? newState.MaterializedViews[newMv.Source].Policies?.HotCache : null))
+                    ?? newState.DefaultRetentionAndCache?.HotCache;
+
+                if(specificCache != null && specificCache.EndsWith("d") && int.TryParse(specificCache.TrimEnd("d"), out int lookBackInDays) && DateTime.TryParse(newMv.EffectiveDateTime, out var effectiveDateTime))
+                {
+                    if(DateTime.UtcNow.AddDays(-lookBackInDays) < effectiveDateTime)
+                    {
+                        // Backfill will work
+                        var validUntil = effectiveDateTime.AddDays(lookBackInDays);
+                        mvChange.Comment = new Comment { FailsRollout = false, Kind = CommentKind.Note, Text = $"The materialized view {mvChange.Entity} is created with backfill configured. All required data is available in hot cache and the rollout is expected to succeed as long as it is rolled out before {validUntil:yyyy-MM-dd HH:mm}UTC. The rollout is executed asynchronusly, depending on the size of the backfill it might take a while." };
+                    }
+                    else
+                    {
+                        // Backfill will fail
+                        var validUntil = DateTime.UtcNow.Date.AddDays(1-lookBackInDays);
+                        mvChange.Comment = new Comment { FailsRollout = true, Kind = CommentKind.Caution, Text = $"Not all data for the backfill of {mvChange.Entity} is available hot. The backfill will fail! Please set the effective Date of the MV to {validUntil:yyyy-MM-dd} or newer." };
+                    }
+                }
+                else
+                {
+                    mvChange.Comment = new Comment { FailsRollout = false, Kind = CommentKind.Warning, Text = $"The conditions for backfilling {mvChange.Entity} couldn't be validated. Please check for errors!" };
+                }
+            }            
+
+            result.AddRange(mvChanges);
             result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.ContinuousExports, nameof(newState.ContinuousExports), log));
             result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.Functions, nameof(newState.Functions), log));
             result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.ExternalTables, nameof(newState.ExternalTables), log));
@@ -63,7 +100,6 @@ public static List<IChange> GenerateChanges(Database oldState, Database newState
                 }
             }
 
-
             return result;
         }
 

KustoSchemaTools/Changes/DeletionChange.cs

@@ -51,5 +51,6 @@ public string Markdown
         }
 
 
+        public Comment Comment { get; set; }
     }
 }

KustoSchemaTools/Changes/FollowerChange.cs

@@ -25,5 +25,6 @@ public BasicChange(string entityType, string entity, string markdown, List<Datab
 
         public List<DatabaseScriptContainer> Scripts { get; set; }
 
+        public Comment Comment { get; set; }
     }
 }

KustoSchemaTools/Changes/Heading.cs

@@ -13,6 +13,7 @@ public Heading(string title)
         public List<DatabaseScriptContainer> Scripts => new List<DatabaseScriptContainer> { };
 
         public string Markdown => $"# {Entity}";
+        public Comment Comment { get; set; }
 
     }
 

KustoSchemaTools/Changes/IChange.cs

@@ -9,6 +9,7 @@ public interface IChange
 
         public string Markdown { get; }
 
-    }
+        public Comment Comment { get; set; }
 
-}
+    }
+}

KustoSchemaTools/KustoSchemaHandler.cs

@@ -47,11 +47,21 @@ public KustoSchemaHandler(ILogger<KustoSchemaHandler<T>> schemaHandlerLogger, Ya
                 var kustoDb = await dbHandler.LoadAsync();
                 var changes = DatabaseChanges.GenerateChanges(kustoDb, yamlDb, escapedDbName, Log);
 
-                isValid &= changes.All(itm => itm.Scripts.All(itm => itm.IsValid != false));
+                var comments = changes.Select(itm => itm.Comment).Where(itm => itm != null).ToList();
+
+        
+                isValid &= changes.All(itm => itm.Scripts.All(itm => itm.IsValid != false)) && comments.All(itm => itm.FailsRollout == false);
 
                 sb.AppendLine($"# {cluster.Name}/{escapedDbName} ({cluster.Url})");
 
-                if(changes.Count == 0)
+                foreach (var comment in comments)
+                {
+                    sb.AppendLine($"> [!{comment.Kind.ToString().ToUpper()}]");
+                    sb.AppendLine($"> {comment.Text}");
+                    sb.AppendLine();
+                }
+
+                if (changes.Count == 0)
                 {
                     sb.AppendLine("No changes detected");
                 }

KustoSchemaTools/Model/MaterializedView.cs

@@ -47,11 +47,11 @@ public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
 
             if (asyncSetup)
             {
-                scripts.Add(new DatabaseScriptContainer("CreateMaterializedView", Kind == "table" ? 40 : 41, $".create async ifnotexists materialized-view with ({properties}) {name} on {Kind} {Source} {{ {Query} }}", true));
+                scripts.Add(new DatabaseScriptContainer("CreateMaterializedViewAsync", Kind == "table" ? 40 : 41, $".create async ifnotexists materialized-view with ({properties}) {name} on {Kind} {Source} {{ {Query} }}", true));
             }
             else
             {
-                scripts.Add(new DatabaseScriptContainer("CreateMaterializedView", Kind == "table" ? 40 : 41, $".create-or-alter materialized-view with ({properties}) {name} on {Kind} {Source} {{ {Query} }}"));
+                scripts.Add(new DatabaseScriptContainer("CreateAlterMaterializedView", Kind == "table" ? 40 : 41, $".create-or-alter materialized-view with ({properties}) {name} on {Kind} {Source} {{ {Query} }}"));
             }
             if (Policies != null)
             {