Skip to content

Commit 862d571

Browse files
caol-ilacaol-ila
authored
Merge pull request #82 from github/caol-ila-partitioning
Add Partitioning Policy
2 parents 1309935 + b0358a1 commit 862d571

8 files changed

Lines changed: 305 additions & 56 deletions

File tree

‎KustoSchemaTools/Model/MaterializedView.cs‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,13 @@ public class MaterializedView : IKustoBaseEntity
1717
public bool? Backfill { get; set; }
1818
public bool AutoUpdateSchema { get; set; } = false;
1919
public List<string> DimensionTables { get; set; }
20+
[Obsolete("Use policies instead")]
2021
public RetentionAndCachePolicy RetentionAndCachePolicy { get; set; } = new RetentionAndCachePolicy();
2122
[YamlMember(ScalarStyle = ScalarStyle.Literal)]
2223
public string Query { get; set; }
24+
[Obsolete("Use policies instead")]
2325
public string? RowLevelSecurity { get; set; }
26+
public Policy? Policies { get; set; }
2427

2528
public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
2629
{
@@ -41,6 +44,7 @@ public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
4144
.Where(p => !string.IsNullOrWhiteSpace(p.Value?.ToString()))
4245
.Select(p => $"{p.Name}=\"{p.Value}\""));
4346

47+
4448
if (asyncSetup)
4549
{
4650
scripts.Add(new DatabaseScriptContainer("CreateMaterializedView", Kind == "table" ? 40 : 41, $".create async ifnotexists materialized-view with ({properties}) {name} on {Kind} {Source} {{ {Query} }}", true));
@@ -49,21 +53,11 @@ public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
4953
{
5054
scripts.Add(new DatabaseScriptContainer("CreateMaterializedView", Kind == "table" ? 40 : 41, $".create-or-alter materialized-view with ({properties}) {name} on {Kind} {Source} {{ {Query} }}"));
5155
}
52-
53-
if (RetentionAndCachePolicy != null)
54-
{
55-
scripts.AddRange(RetentionAndCachePolicy.CreateScripts(name, "materialized-view"));
56-
}
57-
58-
59-
if (!string.IsNullOrEmpty(RowLevelSecurity))
60-
{
61-
scripts.Add(new DatabaseScriptContainer("RowLevelSecurity", 57, $".alter materialized-view {name} policy row_level_security enable \"{RowLevelSecurity}\""));
62-
}
63-
else
56+
if (Policies != null)
6457
{
65-
scripts.Add(new DatabaseScriptContainer("RowLevelSecurity", 52, $".delete materialized-view {name} policy row_level_security"));
58+
scripts.AddRange(Policies.CreateScripts(name, "materialized-view"));
6659
}
60+
6761
return scripts;
6862
}
6963
}

‎KustoSchemaTools/Model/PartitioningPolicy.cs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
using KustoSchemaTools.Changes;
2+
using Newtonsoft.Json;
3+
using Newtonsoft.Json.Linq;
4+
5+
namespace KustoSchemaTools.Model
6+
{
7+
public class PartitioningPolicy
8+
{
9+
public string TimePartitionColumn { get; set; }
10+
public TimeSpan? RangeSize { get; set; }
11+
public bool? OverrideCreationTime { get; set; }
12+
public DateTime? Reference { get; set; }
13+
14+
public string? SecondaryPartition { get; set; }
15+
public PartitionAssignmentMode? PartirionAssignmentMode { get; set; }
16+
public int? MaxPartitionCount { get; set; }
17+
public DateTime? EffectiveDateTime { get; set; }
18+
19+
public DatabaseScriptContainer CreateScript(string name, string entity)
20+
{
21+
22+
var p1 = new
23+
{
24+
ColumnName = TimePartitionColumn,
25+
Kind = "UniformRange",
26+
Properties = new
27+
{
28+
Reference = (Reference ?? DateTime.UtcNow).ToString("yyyy-MM-ddTHH:mm:ss"),
29+
RangeSize = (RangeSize ?? new TimeSpan(1, 0, 0, 0)).ToString(),
30+
OverrideCreationTime = OverrideCreationTime == true
31+
}
32+
};
33+
34+
var p2 = new
35+
{
36+
ColumnName = SecondaryPartition,
37+
Kind = "Hash",
38+
Properties = new
39+
{
40+
Function = "XxHash64",
41+
MaxPartitionCount = MaxPartitionCount ?? 128,
42+
PartitionAssignmentMode = PartirionAssignmentMode ?? PartitionAssignmentMode.Default
43+
}
44+
};
45+
46+
var items = new List<object>();
47+
items.Add(p1);
48+
if(string.IsNullOrWhiteSpace(SecondaryPartition) == false)
49+
{
50+
items.Add(p2);
51+
}
52+
53+
var policy = new
54+
{
55+
EffectiveDateTime = (EffectiveDateTime ?? DateTime.UtcNow).ToString("yyyy-MM-dd"),
56+
PartitionKeys = items
57+
};
58+
59+
60+
return new DatabaseScriptContainer("PartitioningPolicy", 50, $".alter {entity} {name} policy partitioning ```{JsonConvert.SerializeObject(policy, Formatting.None)}```");
61+
}
62+
63+
public enum PartitionAssignmentMode
64+
{
65+
/// <summary>
66+
/// The default assignment mode.
67+
/// </summary>
68+
Default = 0,
69+
70+
/// <summary>
71+
/// Ignore the partition value when assigning a homogeneous extent, and assign them uniformly according to the extent ID.
72+
/// </summary>
73+
Uniform = 1,
74+
75+
/// <summary>
76+
/// Assign homogeneous (partitioned) extents by their partition value, this means assignment ignores extent ID.
77+
/// </summary>
78+
ByPartition = 2,
79+
}
80+
}
81+
}

‎KustoSchemaTools/Model/Policy.cs‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
using KustoSchemaTools.Changes;
2+
using KustoSchemaTools.Helpers;
3+
using Newtonsoft.Json;
4+
5+
namespace KustoSchemaTools.Model
6+
{
7+
public class Policy
8+
{
9+
public string? Retention { get; set; }
10+
public string? HotCache { get; set; }
11+
public PartitioningPolicy? Partitioning { get; set; }
12+
public string? RowLevelSecurity { get; set; }
13+
14+
15+
public List<DatabaseScriptContainer> CreateScripts(string name, string entity)
16+
{
17+
var scripts = new List<DatabaseScriptContainer>();
18+
if (Retention != null)
19+
{
20+
scripts.Add(new DatabaseScriptContainer("SoftDelete", 60, $".alter-merge {entity} {name} policy retention softdelete={Retention}"));
21+
}
22+
if (HotCache != null)
23+
{
24+
scripts.Add(new DatabaseScriptContainer("HotCache", 70, $".alter {entity} {name} policy caching hot={HotCache}"));
25+
}
26+
27+
if (!string.IsNullOrEmpty(RowLevelSecurity))
28+
{
29+
scripts.Add(new DatabaseScriptContainer("RowLevelSecurity", 57, $".alter {entity} {name} policy row_level_security enable ```{RowLevelSecurity}```"));
30+
}
31+
else
32+
{
33+
scripts.Add(new DatabaseScriptContainer("RowLevelSecurity", 52, $".delete {entity} {name} policy row_level_security"));
34+
}
35+
36+
if (Partitioning != null)
37+
{
38+
scripts.Add(Partitioning.CreateScript(name, entity));
39+
}
40+
return scripts;
41+
}
42+
}
43+
44+
public class TablePolicy : Policy
45+
{
46+
47+
public List<UpdatePolicy>? UpdatePolicies { get; set; }
48+
public bool RestrictedViewAccess { get; set; } = false;
49+
public List<DatabaseScriptContainer> CreateScripts(string name)
50+
{
51+
var scripts = new List<DatabaseScriptContainer>();
52+
scripts.AddRange(base.CreateScripts(name, "table"));
53+
if (UpdatePolicies != null)
54+
{
55+
var policies = JsonConvert.SerializeObject(UpdatePolicies, Serialization.JsonPascalCase);
56+
var upPriority = UpdatePolicies.Any() ? 59 : 50;
57+
scripts.Add(new DatabaseScriptContainer("TableUpdatePolicy", upPriority, $".alter table {name} policy update ```{policies}```"));
58+
}
59+
60+
var rvaPrio = RestrictedViewAccess ? 58 : 51;
61+
scripts.Add(new DatabaseScriptContainer("RestrictedViewAccess", rvaPrio, $".alter table {name} policy restricted_view_access {(RestrictedViewAccess ? "true" : "false")}"));
62+
63+
return scripts;
64+
65+
}
66+
}
67+
68+
}

‎KustoSchemaTools/Model/RetentionAndCachePolicy.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,16 @@
22

33
namespace KustoSchemaTools.Model
44
{
5+
[Obsolete("Use policies instead")]
56
public class RetentionAndCachePolicy
67
{
78
public string? Retention { get; set; }
89
public string? HotCache { get; set; }
910

1011
public List<DatabaseScriptContainer> CreateScripts(string name, string entity)
1112
{
13+
if (entity != "database") throw new NotImplementedException();
14+
1215
var scripts = new List<DatabaseScriptContainer>();
1316
if (Retention != null)
1417
{

‎KustoSchemaTools/Model/Table.cs‎

Lines changed: 10 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,18 @@ public class Table : IKustoBaseEntity
99
{
1010

1111
public string Folder { get; set; }
12+
[Obsolete("Use policies instead")]
1213
public RetentionAndCachePolicy RetentionAndCachePolicy { get; set; } = new RetentionAndCachePolicy();
14+
1315
public Dictionary<string, string> Columns { get; set; }
16+
[Obsolete("Use policies instead")]
1417
public List<UpdatePolicy> UpdatePolicies { get; set; }
18+
public TablePolicy? Policies { get; set; }
1519
public List<DatabaseScript> Scripts { get; set; }
1620
public string DocString { get; set; }
17-
public bool RestrictedViewAccess { get; set; } = false;
21+
[Obsolete("Use policies instead")]
1822
public string? RowLevelSecurity { get; set; }
19-
23+
2024
public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
2125
{
2226
var scripts = new List<DatabaseScriptContainer>();
@@ -28,39 +32,20 @@ public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
2832

2933
scripts.Add(new DatabaseScriptContainer("CreateMergeTable", 30, $".create-merge table {name} ({string.Join(", ", Columns.Select(c => $"{c.Key.BracketIfIdentifier()}:{c.Value}"))})"));
3034
}
31-
else
32-
{
33-
34-
}
3535

3636
scripts.Add(new DatabaseScriptContainer("TableFolder", 31, $".alter table {name} folder '{Folder}'"));
3737
scripts.Add(new DatabaseScriptContainer("TableDocString", 31, $".alter table {name} docstring '{DocString}'"));
38-
var ups = UpdatePolicies ?? new List<UpdatePolicy>();
39-
var policies = JsonConvert.SerializeObject(ups, Serialization.JsonPascalCase);
40-
var upPriority = ups.Any() ? 59 : 50;
41-
scripts.Add(new DatabaseScriptContainer("TableUpdatePolicy", upPriority, $".alter table {name} policy update ```{policies}```"));
38+
4239

43-
if (RetentionAndCachePolicy != null)
40+
if (Policies != null)
4441
{
45-
scripts.AddRange(RetentionAndCachePolicy.CreateScripts(name, "table"));
42+
scripts.AddRange(Policies.CreateScripts(name));
4643
}
4744
if (Scripts != null)
4845
{
4946
scripts.AddRange(Scripts.Select(itm => new DatabaseScriptContainer(itm, "DatabaseScript")));
5047
}
51-
52-
var rvaPrio = RestrictedViewAccess ? 58 : 51;
53-
scripts.Add(new DatabaseScriptContainer("RestrictedViewAccess", rvaPrio, $".alter table {name} policy restricted_view_access {(RestrictedViewAccess ? "true" : "false")}"));
54-
55-
56-
if (!string.IsNullOrEmpty(RowLevelSecurity))
57-
{
58-
scripts.Add(new DatabaseScriptContainer("RowLevelSecurity", 57, $".alter table {name} policy row_level_security enable ```{RowLevelSecurity}```"));
59-
}
60-
else
61-
{
62-
scripts.Add(new DatabaseScriptContainer("RowLevelSecurity", 52, $".delete table {name} policy row_level_security"));
63-
}
48+
6449
return scripts;
6550
}
6651
}

0 commit comments

Comments
 (0)