Skip to content

Commit 6d0bfad

Browse files
bladata1990claude
bladata1990
and
claude
committed
Add E2E test for STRUCT/ARRAY/MAP nested column expansion (CSA-371)
Adds ComplexTypeColumnsRABTest covering the full pipeline from CSV input to nested child columns in Atlan, verifying parentColumnQualifiedName, columnDepthLevel, synthetic QN nodes (/items/, /values/), and depth-2 recursion. Also updates assets-complex.csv with adminRoles/adminUsers so the connection can be created during test setup. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> Signed-off-by: bladata1990 <balakrishnan.r@atlan.com>
1 parent 7a969c6 commit 6d0bfad

2 files changed

Lines changed: 290 additions & 10 deletions

File tree

‎samples/packages/relational-assets-builder/src/test/kotlin/com/atlan/pkg/rab/ComplexTypeColumnsRABTest.kt‎

Lines changed: 280 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,280 @@
1+
/* SPDX-License-Identifier: Apache-2.0
2+
Copyright 2023 Atlan Pte. Ltd. */
3+
package com.atlan.pkg.rab
4+
5+
import AssetImportCfg
6+
import RelationalAssetsBuilderCfg
7+
import com.atlan.model.assets.Asset
8+
import com.atlan.model.assets.Column
9+
import com.atlan.model.assets.Connection
10+
import com.atlan.model.assets.Table
11+
import com.atlan.model.enums.AtlanConnectorType
12+
import com.atlan.model.fields.AtlanField
13+
import com.atlan.pkg.PackageTest
14+
import com.atlan.pkg.Utils
15+
import com.atlan.pkg.rab.Importer.PREVIOUS_FILES_PREFIX
16+
import org.testng.Assert.assertFalse
17+
import org.testng.Assert.assertTrue
18+
import java.io.File
19+
import java.nio.file.Paths
20+
import kotlin.test.Test
21+
import kotlin.test.assertEquals
22+
import kotlin.test.assertNotNull
23+
24+
/**
25+
* Test that RAB correctly expands STRUCT/ARRAY/MAP complex-type columns into nested child columns.
26+
*/
27+
class ComplexTypeColumnsRABTest : PackageTest("ctcol") {
28+
override val logger = Utils.getLogger(this.javaClass.name)
29+
30+
private val conn1 = makeUnique("c1")
31+
private val conn1Type = AtlanConnectorType.ICEBERG
32+
33+
private val testFile = "input.csv"
34+
private val files = listOf(testFile, "debug.log")
35+
36+
private fun prepFile() {
37+
val input = Paths.get("src", "test", "resources", "assets-complex.csv").toFile()
38+
val output = Paths.get(testDirectory, testFile).toFile()
39+
input.useLines { lines ->
40+
lines.forEach { line ->
41+
val revised =
42+
line
43+
.replace("{{CONNECTION1}}", conn1)
44+
.replace("{{API_TOKEN_USER}}", client.users.currentUser.username)
45+
output.appendText("$revised\n")
46+
}
47+
}
48+
}
49+
50+
private val connectionAttrs: List<AtlanField> =
51+
listOf(
52+
Connection.NAME,
53+
Connection.CONNECTOR_TYPE,
54+
)
55+
56+
private val tableAttrs: List<AtlanField> =
57+
listOf(
58+
Table.NAME,
59+
Table.CONNECTION_QUALIFIED_NAME,
60+
Table.COLUMN_COUNT,
61+
Table.COLUMNS,
62+
)
63+
64+
private val columnAttrs: List<AtlanField> =
65+
listOf(
66+
Column.NAME,
67+
Column.STATUS,
68+
Column.CONNECTION_QUALIFIED_NAME,
69+
Column.TABLE_NAME,
70+
Column.TABLE_QUALIFIED_NAME,
71+
Column.DATA_TYPE,
72+
Column.RAW_DATA_TYPE_DEFINITION,
73+
Column.ORDER,
74+
Column.PARENT_COLUMN_QUALIFIED_NAME,
75+
Column.PARENT_COLUMN_NAME,
76+
Column.COLUMN_DEPTH_LEVEL,
77+
Column.COLUMN_HIERARCHY,
78+
Column.NESTED_COLUMN_ORDER,
79+
)
80+
81+
override fun setup() {
82+
prepFile()
83+
runCustomPackage(
84+
RelationalAssetsBuilderCfg(
85+
assetsFile = Paths.get(testDirectory, testFile).toString(),
86+
assetsUpsertSemantic = "upsert",
87+
assetsFailOnErrors = true,
88+
deltaSemantic = "full",
89+
),
90+
Importer::main,
91+
)
92+
runCustomPackage(
93+
AssetImportCfg(
94+
assetsFile = "$testDirectory${File.separator}current-file-transformed.csv",
95+
assetsUpsertSemantic = "upsert",
96+
assetsDeltaSemantic = "full",
97+
assetsFailOnErrors = true,
98+
assetsPreviousFilePrefix = PREVIOUS_FILES_PREFIX,
99+
),
100+
com.atlan.pkg.aim.Importer::main,
101+
)
102+
}
103+
104+
override fun teardown() {
105+
removeConnection(conn1, conn1Type)
106+
}
107+
108+
@Test(groups = ["rab.ctcol.create"])
109+
fun tableHasOnlyTopLevelColumns() {
110+
// The table_columns relationship must only contain the 5 top-level columns;
111+
// nested child columns have their table reference cleared by ColumnXformer.
112+
val c1 = Connection.findByName(client, conn1, conn1Type, connectionAttrs)[0]!!
113+
val request =
114+
Table
115+
.select(client)
116+
.where(Table.CONNECTION_QUALIFIED_NAME.eq(c1.qualifiedName))
117+
.includesOnResults(tableAttrs)
118+
.includeOnRelations(Asset.NAME)
119+
.toRequest()
120+
val response = retrySearchUntil(request, 1)
121+
val tbl = response.assets[0] as Table
122+
assertEquals("TEST_TBL", tbl.name)
123+
assertEquals(5, tbl.columnCount)
124+
val topLevelNames = tbl.columns.map { it.name }.toSet()
125+
assertEquals(setOf("PLAIN_COL", "STRUCT_COL", "ARRAY_COL", "MAP_COL", "NESTED_STRUCT_COL"), topLevelNames)
126+
}
127+
128+
@Test(groups = ["rab.ctcol.create"])
129+
fun structColChildColumns() {
130+
// STRUCT<city:STRING, zip:INT> → two direct children at depth 1
131+
val c1 = Connection.findByName(client, conn1, conn1Type, connectionAttrs)[0]!!
132+
val structColQN = "${c1.qualifiedName}/TEST_DB/TEST_SCHEMA/TEST_TBL/STRUCT_COL"
133+
val request =
134+
Column
135+
.select(client)
136+
.where(Column.PARENT_COLUMN_QUALIFIED_NAME.eq(structColQN))
137+
.includesOnResults(columnAttrs)
138+
.toRequest()
139+
val response = retrySearchUntil(request, 2)
140+
val found = response.assets
141+
assertEquals(2, found.size)
142+
assertEquals(setOf("city", "zip"), found.map { it.name }.toSet())
143+
found.forEach { asset ->
144+
val col = asset as Column
145+
assertEquals(structColQN, col.parentColumnQualifiedName)
146+
assertEquals("STRUCT_COL", col.parentColumnName)
147+
assertEquals(1, col.columnDepthLevel)
148+
assertNotNull(col.columnHierarchy)
149+
assertFalse(col.columnHierarchy.isEmpty())
150+
assertTrue(col.tableName.isNullOrEmpty())
151+
assertTrue(col.tableQualifiedName.isNullOrEmpty())
152+
when (col.name) {
153+
"city" -> {
154+
assertEquals("STRING", col.dataType)
155+
assertEquals("1", col.nestedColumnOrder)
156+
}
157+
158+
"zip" -> {
159+
assertEquals("INT", col.dataType)
160+
assertEquals("2", col.nestedColumnOrder)
161+
}
162+
}
163+
}
164+
}
165+
166+
@Test(groups = ["rab.ctcol.create"])
167+
fun arrayColChildColumns() {
168+
// ARRAY<STRUCT<id:INT, name:STRING>> → two children whose QN passes through /items/ synthetic node
169+
// but whose parentColumnQualifiedName points directly at ARRAY_COL
170+
val c1 = Connection.findByName(client, conn1, conn1Type, connectionAttrs)[0]!!
171+
val arrayColQN = "${c1.qualifiedName}/TEST_DB/TEST_SCHEMA/TEST_TBL/ARRAY_COL"
172+
val request =
173+
Column
174+
.select(client)
175+
.where(Column.PARENT_COLUMN_QUALIFIED_NAME.eq(arrayColQN))
176+
.includesOnResults(columnAttrs)
177+
.toRequest()
178+
val response = retrySearchUntil(request, 2)
179+
val found = response.assets
180+
assertEquals(2, found.size)
181+
assertEquals(setOf("id", "name"), found.map { it.name }.toSet())
182+
found.forEach { asset ->
183+
val col = asset as Column
184+
assertEquals(arrayColQN, col.parentColumnQualifiedName)
185+
assertEquals(1, col.columnDepthLevel)
186+
// QN must route through the synthetic "items" node
187+
assertTrue(col.qualifiedName.contains("/items/"), "Expected /items/ in QN: ${col.qualifiedName}")
188+
assertTrue(col.tableName.isNullOrEmpty())
189+
}
190+
}
191+
192+
@Test(groups = ["rab.ctcol.create"])
193+
fun mapColChildColumns() {
194+
// MAP<STRING, STRUCT<key:STRING, value:DOUBLE>> → two children whose QN passes through /values/ synthetic node
195+
val c1 = Connection.findByName(client, conn1, conn1Type, connectionAttrs)[0]!!
196+
val mapColQN = "${c1.qualifiedName}/TEST_DB/TEST_SCHEMA/TEST_TBL/MAP_COL"
197+
val request =
198+
Column
199+
.select(client)
200+
.where(Column.PARENT_COLUMN_QUALIFIED_NAME.eq(mapColQN))
201+
.includesOnResults(columnAttrs)
202+
.toRequest()
203+
val response = retrySearchUntil(request, 2)
204+
val found = response.assets
205+
assertEquals(2, found.size)
206+
assertEquals(setOf("key", "value"), found.map { it.name }.toSet())
207+
found.forEach { asset ->
208+
val col = asset as Column
209+
assertEquals(mapColQN, col.parentColumnQualifiedName)
210+
assertEquals(1, col.columnDepthLevel)
211+
// QN must route through the synthetic "values" node
212+
assertTrue(col.qualifiedName.contains("/values/"), "Expected /values/ in QN: ${col.qualifiedName}")
213+
assertTrue(col.tableName.isNullOrEmpty())
214+
}
215+
}
216+
217+
@Test(groups = ["rab.ctcol.create"])
218+
fun nestedStructColDepth1Children() {
219+
// STRUCT<outer:STRUCT<...>, label:STRING> → two depth-1 children directly under NESTED_STRUCT_COL
220+
val c1 = Connection.findByName(client, conn1, conn1Type, connectionAttrs)[0]!!
221+
val nestedColQN = "${c1.qualifiedName}/TEST_DB/TEST_SCHEMA/TEST_TBL/NESTED_STRUCT_COL"
222+
val request =
223+
Column
224+
.select(client)
225+
.where(Column.PARENT_COLUMN_QUALIFIED_NAME.eq(nestedColQN))
226+
.includesOnResults(columnAttrs)
227+
.toRequest()
228+
val response = retrySearchUntil(request, 2)
229+
val found = response.assets
230+
assertEquals(2, found.size)
231+
assertEquals(setOf("outer", "label"), found.map { it.name }.toSet())
232+
found.forEach { asset ->
233+
val col = asset as Column
234+
assertEquals(nestedColQN, col.parentColumnQualifiedName)
235+
assertEquals(1, col.columnDepthLevel)
236+
assertTrue(col.tableName.isNullOrEmpty())
237+
when (col.name) {
238+
"outer" -> assertEquals("STRUCT", col.dataType)
239+
"label" -> assertEquals("STRING", col.dataType)
240+
}
241+
}
242+
}
243+
244+
@Test(groups = ["rab.ctcol.create"])
245+
fun nestedStructColDepth2Children() {
246+
// The STRUCT-typed "outer" field expands to inner:STRING, count:INT at depth 2
247+
val c1 = Connection.findByName(client, conn1, conn1Type, connectionAttrs)[0]!!
248+
val outerColQN = "${c1.qualifiedName}/TEST_DB/TEST_SCHEMA/TEST_TBL/NESTED_STRUCT_COL/outer"
249+
val request =
250+
Column
251+
.select(client)
252+
.where(Column.PARENT_COLUMN_QUALIFIED_NAME.eq(outerColQN))
253+
.includesOnResults(columnAttrs)
254+
.toRequest()
255+
val response = retrySearchUntil(request, 2)
256+
val found = response.assets
257+
assertEquals(2, found.size)
258+
assertEquals(setOf("inner", "count"), found.map { it.name }.toSet())
259+
found.forEach { asset ->
260+
val col = asset as Column
261+
assertEquals(outerColQN, col.parentColumnQualifiedName)
262+
assertEquals("outer", col.parentColumnName)
263+
assertEquals(2, col.columnDepthLevel)
264+
// Hierarchy must list both NESTED_STRUCT_COL (depth 1) and outer (depth 2)
265+
assertNotNull(col.columnHierarchy)
266+
assertEquals(2, col.columnHierarchy.size)
267+
assertTrue(col.tableName.isNullOrEmpty())
268+
}
269+
}
270+
271+
@Test(dependsOnGroups = ["rab.ctcol.create"])
272+
fun filesCreated() {
273+
validateFilesExist(files)
274+
}
275+
276+
@Test(dependsOnGroups = ["rab.ctcol.create"])
277+
fun errorFreeLog() {
278+
validateErrorFreeLog()
279+
}
280+
}

‎samples/packages/relational-assets-builder/src/test/resources/assets-complex.csv‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
typeName,connectionName,connectorType,databaseName,schemaName,entityName,columnName,dataType,displayName,description
2-
Connection,{{CONNECTION1}},iceberg,,,,,,,
3-
Database,{{CONNECTION1}},iceberg,TEST_DB,,,,,,
4-
Schema,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,,,,,
5-
Table,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,TEST_TBL,,,,
6-
Column,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,TEST_TBL,PLAIN_COL,VARCHAR(128),Plain column,A regular column
7-
Column,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,TEST_TBL,STRUCT_COL,"STRUCT<city:STRING, zip:INT>",Struct column,A STRUCT typed column
8-
Column,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,TEST_TBL,ARRAY_COL,"ARRAY<STRUCT<id:INT, name:STRING>>",Array column,An ARRAY of STRUCT typed column
9-
Column,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,TEST_TBL,MAP_COL,"MAP<STRING, STRUCT<key:STRING, value:DOUBLE>>",Map column,A MAP with STRUCT value typed column
10-
Column,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,TEST_TBL,NESTED_STRUCT_COL,"STRUCT<outer:STRUCT<inner:STRING, count:INT>, label:STRING>",Nested struct column,A STRUCT containing another STRUCT
1+
typeName,connectionName,connectorType,databaseName,schemaName,entityName,columnName,dataType,displayName,description,adminRoles,adminGroups,adminUsers
2+
Connection,{{CONNECTION1}},iceberg,,,,,,,,$admin,,{{API_TOKEN_USER}}
3+
Database,{{CONNECTION1}},iceberg,TEST_DB,,,,,,,,,
4+
Schema,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,,,,,,,,
5+
Table,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,TEST_TBL,,,,,,,
6+
Column,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,TEST_TBL,PLAIN_COL,VARCHAR(128),Plain column,A regular column,,,
7+
Column,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,TEST_TBL,STRUCT_COL,"STRUCT<city:STRING, zip:INT>",Struct column,A STRUCT typed column,,,
8+
Column,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,TEST_TBL,ARRAY_COL,"ARRAY<STRUCT<id:INT, name:STRING>>",Array column,An ARRAY of STRUCT typed column,,,
9+
Column,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,TEST_TBL,MAP_COL,"MAP<STRING, STRUCT<key:STRING, value:DOUBLE>>",Map column,A MAP with STRUCT value typed column,,,
10+
Column,{{CONNECTION1}},iceberg,TEST_DB,TEST_SCHEMA,TEST_TBL,NESTED_STRUCT_COL,"STRUCT<outer:STRUCT<inner:STRING, count:INT>, label:STRING>",Nested struct column,A STRUCT containing another STRUCT,,,

0 commit comments

Comments
 (0)