Skip to content

Commit a29950e

Browse files
Add TableProvisionJob CRD and JobTemplate support for CREATE TABLE
1 parent de62fbd commit a29950e

9 files changed

Lines changed: 796 additions & 1 deletion

generate-models.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ docker run \
1616
-u "$(pwd)/hoptimator-k8s/src/main/resources/pipelines.crd.yaml" \
1717
-u "$(pwd)/hoptimator-k8s/src/main/resources/sqljobs.crd.yaml" \
1818
-u "$(pwd)/hoptimator-k8s/src/main/resources/subscriptions.crd.yaml" \
19+
-u "$(pwd)/hoptimator-k8s/src/main/resources/tableprovisionjobs.crd.yaml" \
1920
-u "$(pwd)/hoptimator-k8s/src/main/resources/tabletemplates.crd.yaml" \
2021
-u "$(pwd)/hoptimator-k8s/src/main/resources/tabletriggers.crd.yaml" \
2122
-u "$(pwd)/hoptimator-k8s/src/main/resources/views.crd.yaml" \

hoptimator-k8s/generate-models.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@ docker run \
1010
--network host \
1111
ghcr.io/kubernetes-client/java/crd-model-gen:v1.0.6 \
1212
/generate.sh -o "$(pwd)" -n "" -p "com.linkedin.hoptimator.k8s" \
13-
-u "$(pwd)/src/main/resources/tabletriggers.crd.yaml" \
13+
-u "$(pwd)/src/main/resources/tableprovisionjobs.crd.yaml" \
1414
&& echo "done."

hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sApiEndpoints.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
import com.linkedin.hoptimator.k8s.models.V1alpha1JobTemplateList;
1818
import com.linkedin.hoptimator.k8s.models.V1alpha1Pipeline;
1919
import com.linkedin.hoptimator.k8s.models.V1alpha1PipelineList;
20+
import com.linkedin.hoptimator.k8s.models.V1alpha1TableProvisionJob;
21+
import com.linkedin.hoptimator.k8s.models.V1alpha1TableProvisionJobList;
2022
import com.linkedin.hoptimator.k8s.models.V1alpha1TableTemplate;
2123
import com.linkedin.hoptimator.k8s.models.V1alpha1TableTemplateList;
2224
import com.linkedin.hoptimator.k8s.models.V1alpha1TableTrigger;
@@ -58,6 +60,9 @@ public final class K8sApiEndpoints {
5860
public static final K8sApiEndpoint<V1alpha1JobTemplate, V1alpha1JobTemplateList> JOB_TEMPLATES =
5961
new K8sApiEndpoint<>("JobTemplate", "hoptimator.linkedin.com", "v1alpha1", "jobtemplates", false,
6062
V1alpha1JobTemplate.class, V1alpha1JobTemplateList.class);
63+
public static final K8sApiEndpoint<V1alpha1TableProvisionJob, V1alpha1TableProvisionJobList> TABLE_PROVISION_JOBS =
64+
new K8sApiEndpoint<>("TableProvisionJob", "hoptimator.linkedin.com", "v1alpha1",
65+
"tableprovisionjobs", false, V1alpha1TableProvisionJob.class, V1alpha1TableProvisionJobList.class);
6166
public static final K8sApiEndpoint<V1alpha1TableTrigger, V1alpha1TableTriggerList> TABLE_TRIGGERS =
6267
new K8sApiEndpoint<>("TableTrigger", "hoptimator.linkedin.com", "v1alpha1", "tabletriggers", false,
6368
V1alpha1TableTrigger.class, V1alpha1TableTriggerList.class);

hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDeployerProvider.java

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,20 @@
33
import java.sql.Connection;
44
import java.util.ArrayList;
55
import java.util.Collection;
6+
import java.util.Collections;
7+
import java.util.HashMap;
68
import java.util.List;
9+
import java.util.Map;
710

811
import com.linkedin.hoptimator.Deployable;
912
import com.linkedin.hoptimator.Deployer;
1013
import com.linkedin.hoptimator.DeployerProvider;
1114
import com.linkedin.hoptimator.Job;
1215
import com.linkedin.hoptimator.MaterializedView;
16+
import com.linkedin.hoptimator.Sink;
1317
import com.linkedin.hoptimator.Source;
18+
import com.linkedin.hoptimator.SqlDialect;
19+
import com.linkedin.hoptimator.ThrowingFunction;
1420
import com.linkedin.hoptimator.Trigger;
1521
import com.linkedin.hoptimator.View;
1622

@@ -29,13 +35,26 @@ public <T extends Deployable> Collection<Deployer> deployers(T obj, Connection c
2935
list.add(new K8sJobDeployer((Job) obj, context));
3036
} else if (obj instanceof Source) {
3137
list.add(new K8sSourceDeployer((Source) obj, context));
38+
if (!(obj instanceof Sink)) {
39+
// Sets up a no-op table provisioning job deployer.
40+
list.add(new K8sJobDeployer(jobFromSource((Source) obj), context));
41+
}
3242
} else if (obj instanceof Trigger) {
3343
list.add(new K8sTriggerDeployer((Trigger) obj, context));
3444
}
3545

3646
return list;
3747
}
3848

49+
private static Job jobFromSource(Source source) {
50+
Sink sink = new Sink(source.database(), source.path(), source.options());
51+
Map<String, ThrowingFunction<SqlDialect, String>> lazyEvals = new HashMap<>();
52+
lazyEvals.put("sql", dialect -> null);
53+
lazyEvals.put("query", dialect -> null);
54+
lazyEvals.put("fieldMap", dialect -> null);
55+
return new Job(source.table(), Collections.emptySet(), sink, lazyEvals);
56+
}
57+
3958
@Override
4059
public int priority() {
4160
return 1;
Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
/*
2+
* Kubernetes
3+
* No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
4+
*
5+
* The version of the OpenAPI document: v1.21.1
6+
*
7+
*
8+
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
9+
* https://openapi-generator.tech
10+
* Do not edit the class manually.
11+
*/
12+
13+
14+
package com.linkedin.hoptimator.k8s.models;
15+
16+
import java.util.Objects;
17+
import java.util.Arrays;
18+
import com.google.gson.TypeAdapter;
19+
import com.google.gson.annotations.JsonAdapter;
20+
import com.google.gson.annotations.SerializedName;
21+
import com.google.gson.stream.JsonReader;
22+
import com.google.gson.stream.JsonWriter;
23+
import com.linkedin.hoptimator.k8s.models.V1alpha1TableProvisionJobSpec;
24+
import com.linkedin.hoptimator.k8s.models.V1alpha1TableProvisionJobStatus;
25+
import io.kubernetes.client.openapi.models.V1ObjectMeta;
26+
import io.swagger.annotations.ApiModel;
27+
import io.swagger.annotations.ApiModelProperty;
28+
import java.io.IOException;
29+
30+
/**
31+
* Job to provision backing infrastructure for a table.
32+
*/
33+
@ApiModel(description = "Job to provision backing infrastructure for a table.")
34+
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2026-03-13T23:58:00.641Z[Etc/UTC]")
35+
public class V1alpha1TableProvisionJob implements io.kubernetes.client.common.KubernetesObject {
36+
public static final String SERIALIZED_NAME_API_VERSION = "apiVersion";
37+
@SerializedName(SERIALIZED_NAME_API_VERSION)
38+
private String apiVersion;
39+
40+
public static final String SERIALIZED_NAME_KIND = "kind";
41+
@SerializedName(SERIALIZED_NAME_KIND)
42+
private String kind;
43+
44+
public static final String SERIALIZED_NAME_METADATA = "metadata";
45+
@SerializedName(SERIALIZED_NAME_METADATA)
46+
private V1ObjectMeta metadata = null;
47+
48+
public static final String SERIALIZED_NAME_SPEC = "spec";
49+
@SerializedName(SERIALIZED_NAME_SPEC)
50+
private V1alpha1TableProvisionJobSpec spec;
51+
52+
public static final String SERIALIZED_NAME_STATUS = "status";
53+
@SerializedName(SERIALIZED_NAME_STATUS)
54+
private V1alpha1TableProvisionJobStatus status;
55+
56+
57+
public V1alpha1TableProvisionJob apiVersion(String apiVersion) {
58+
59+
this.apiVersion = apiVersion;
60+
return this;
61+
}
62+
63+
/**
64+
* APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
65+
* @return apiVersion
66+
**/
67+
@javax.annotation.Nullable
68+
@ApiModelProperty(value = "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources")
69+
70+
public String getApiVersion() {
71+
return apiVersion;
72+
}
73+
74+
75+
public void setApiVersion(String apiVersion) {
76+
this.apiVersion = apiVersion;
77+
}
78+
79+
80+
public V1alpha1TableProvisionJob kind(String kind) {
81+
82+
this.kind = kind;
83+
return this;
84+
}
85+
86+
/**
87+
* Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
88+
* @return kind
89+
**/
90+
@javax.annotation.Nullable
91+
@ApiModelProperty(value = "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds")
92+
93+
public String getKind() {
94+
return kind;
95+
}
96+
97+
98+
public void setKind(String kind) {
99+
this.kind = kind;
100+
}
101+
102+
103+
public V1alpha1TableProvisionJob metadata(V1ObjectMeta metadata) {
104+
105+
this.metadata = metadata;
106+
return this;
107+
}
108+
109+
/**
110+
* Get metadata
111+
* @return metadata
112+
**/
113+
@javax.annotation.Nullable
114+
@ApiModelProperty(value = "")
115+
116+
public V1ObjectMeta getMetadata() {
117+
return metadata;
118+
}
119+
120+
121+
public void setMetadata(V1ObjectMeta metadata) {
122+
this.metadata = metadata;
123+
}
124+
125+
126+
public V1alpha1TableProvisionJob spec(V1alpha1TableProvisionJobSpec spec) {
127+
128+
this.spec = spec;
129+
return this;
130+
}
131+
132+
/**
133+
* Get spec
134+
* @return spec
135+
**/
136+
@javax.annotation.Nullable
137+
@ApiModelProperty(value = "")
138+
139+
public V1alpha1TableProvisionJobSpec getSpec() {
140+
return spec;
141+
}
142+
143+
144+
public void setSpec(V1alpha1TableProvisionJobSpec spec) {
145+
this.spec = spec;
146+
}
147+
148+
149+
public V1alpha1TableProvisionJob status(V1alpha1TableProvisionJobStatus status) {
150+
151+
this.status = status;
152+
return this;
153+
}
154+
155+
/**
156+
* Get status
157+
* @return status
158+
**/
159+
@javax.annotation.Nullable
160+
@ApiModelProperty(value = "")
161+
162+
public V1alpha1TableProvisionJobStatus getStatus() {
163+
return status;
164+
}
165+
166+
167+
public void setStatus(V1alpha1TableProvisionJobStatus status) {
168+
this.status = status;
169+
}
170+
171+
172+
@Override
173+
public boolean equals(Object o) {
174+
if (this == o) {
175+
return true;
176+
}
177+
if (o == null || getClass() != o.getClass()) {
178+
return false;
179+
}
180+
V1alpha1TableProvisionJob v1alpha1TableProvisionJob = (V1alpha1TableProvisionJob) o;
181+
return Objects.equals(this.apiVersion, v1alpha1TableProvisionJob.apiVersion) &&
182+
Objects.equals(this.kind, v1alpha1TableProvisionJob.kind) &&
183+
Objects.equals(this.metadata, v1alpha1TableProvisionJob.metadata) &&
184+
Objects.equals(this.spec, v1alpha1TableProvisionJob.spec) &&
185+
Objects.equals(this.status, v1alpha1TableProvisionJob.status);
186+
}
187+
188+
@Override
189+
public int hashCode() {
190+
return Objects.hash(apiVersion, kind, metadata, spec, status);
191+
}
192+
193+
194+
@Override
195+
public String toString() {
196+
StringBuilder sb = new StringBuilder();
197+
sb.append("class V1alpha1TableProvisionJob {\n");
198+
sb.append(" apiVersion: ").append(toIndentedString(apiVersion)).append("\n");
199+
sb.append(" kind: ").append(toIndentedString(kind)).append("\n");
200+
sb.append(" metadata: ").append(toIndentedString(metadata)).append("\n");
201+
sb.append(" spec: ").append(toIndentedString(spec)).append("\n");
202+
sb.append(" status: ").append(toIndentedString(status)).append("\n");
203+
sb.append("}");
204+
return sb.toString();
205+
}
206+
207+
/**
208+
* Convert the given object to string with each line indented by 4 spaces
209+
* (except the first line).
210+
*/
211+
private String toIndentedString(Object o) {
212+
if (o == null) {
213+
return "null";
214+
}
215+
return o.toString().replace("\n", "\n ");
216+
}
217+
218+
}
219+

0 commit comments

Comments
 (0)