Merge pull request #35 from cloudsufi/override-schema-release

vikasrathee-cs · web-flow · commit 82b77d1aaf2c · 2024-10-03T12:10:52.000+05:30
[cherrypick][PLUGIN-1809]Fix added to override schema
diff --git a/.github/workflows/build-report.yml b/.github/workflows/build-report.yml
@@ -0,0 +1,50 @@
+# Copyright © 2024 Cask Data, Inc.
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may not
+#  use this file except in compliance with the License. You may obtain a copy of
+#  the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations under
+#  the License.
+
+# This workflow will build a Java project with Maven
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# Note: Any changes to this workflow would be used only after merging into develop
+name: Build Unit Tests Report
+
+on:
+  workflow_run:
+    workflows:
+      - Build with unit tests
+    types:
+      - completed
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    if: ${{ github.event.workflow_run.conclusion != 'skipped' }}
+
+    steps:
+      # Pinned 1.0.0 version
+      - uses: marocchino/action-workflow_run-status@54b6e87d6cb552fc5f36dbe9a722a6048725917a
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          path: artifacts/
+
+      - name: Surefire Report
+        # Pinned 3.5.2 version
+        uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32
+        if: always()
+        with:
+          report_paths: '**/target/surefire-reports/TEST-*.xml'
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          detailed_summary: true
+          commit: ${{ github.event.workflow_run.head_sha }}
+          check_name: Build Test Report
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -0,0 +1,59 @@
+# Copyright © 2024 Cask Data, Inc.
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may not
+#  use this file except in compliance with the License. You may obtain a copy of
+#  the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations under
+#  the License.
+
+# This workflow will build a Java project with Maven
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# Note: Any changes to this workflow would be used only after merging into develop
+name: Build with unit tests
+
+on:
+  push:
+    branches: [ develop, release/** ]
+  pull_request:
+    branches: [ develop, release/** ]
+    types: [opened, synchronize, reopened, labeled]
+
+jobs:
+  build:
+    runs-on: k8s-runner-build
+
+    # We allow builds:
+    # 1) When it's a merge into a branch
+    # 2) For PRs that are labeled as build and
+    #  - It's a code change
+    #  - A build label was just added
+    # A bit complex, but prevents builds when other labels are manipulated
+    if: >
+      github.event_name == 'push'
+      || (contains(github.event.pull_request.labels.*.name, 'build')
+         && (github.event.action != 'labeled' || github.event.label.name == 'build')
+         )
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.workflow_run.head_sha }}
+      - name: Cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-${{ github.workflow }}
+      - name: Build with Maven
+        run: mvn clean test -fae -T 2 -B -V -DcloudBuild -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+      - name: Archive build artifacts
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: reports-${{ github.run_id }}
+          path: |
+            **/target/rat.txt
+            **/target/surefire-reports/*
diff --git a/pom.xml b/pom.xml
@@ -29,8 +29,8 @@
     <surefire.redirectTestOutputToFile>true</surefire.redirectTestOutputToFile>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <!-- version properties -->
-    <cdap.version>6.1.0-SNAPSHOT</cdap.version>
-    <hydrator.version>2.3.0-SNAPSHOT</hydrator.version>
+    <cdap.version>6.4.1</cdap.version>
+    <hydrator.version>2.6.1</hydrator.version>
     <commons.csv.version>1.6</commons.csv.version>
     <hadoop.version>2.8.0</hadoop.version>
     <spark2.version>2.3.1</spark2.version>
@@ -398,8 +398,8 @@
           <version>1.1.0</version>
           <configuration>
             <cdapArtifacts>
-              <parent>system:cdap-data-pipeline[6.1.0-SNAPSHOT,7.0.0-SNAPSHOT)</parent>
-              <parent>system:cdap-data-streams[6.1.0-SNAPSHOT,7.0.0-SNAPSHOT)</parent>
+              <parent>system:cdap-data-pipeline[6.4.1-SNAPSHOT,7.0.0-SNAPSHOT)</parent>
+              <parent>system:cdap-data-streams[6.4.1-SNAPSHOT,7.0.0-SNAPSHOT)</parent>
             </cdapArtifacts>
           </configuration>
           <executions>
diff --git a/src/main/java/io/cdap/plugin/snowflake/common/util/QueryUtil.java b/src/main/java/io/cdap/plugin/snowflake/common/util/QueryUtil.java
@@ -22,7 +22,7 @@
 public class QueryUtil {
 
   // Matches "limit <number>". Also "limit $$$$" and "limit ''" which means unlimited in Snowflake.
-  private static final String LIMIT_PATTERN = "(?i)LIMIT (''|\\$\\$\\$\\$|\\d+)";
+  private static final String LIMIT_PATTERN = "(?i)LIMIT (NULL|''|\\$\\$\\$\\$|\\d+)";
   private static final String LIMIT_STRING = "limit %s";
 
   private QueryUtil() {
diff --git a/src/main/java/io/cdap/plugin/snowflake/source/batch/SnowflakeBatchSource.java b/src/main/java/io/cdap/plugin/snowflake/source/batch/SnowflakeBatchSource.java
@@ -87,8 +87,7 @@ public void prepareRun(BatchSourceContext context) {
   @Override
   public void initialize(BatchRuntimeContext context) throws Exception {
     super.initialize(context);
-    SnowflakeSourceAccessor snowflakeAccessor = new SnowflakeSourceAccessor(config);
-    Schema schema = SchemaHelper.getSchema(snowflakeAccessor, config.getImportQuery());
+    Schema schema = SchemaHelper.getSchema(config, context.getFailureCollector());
     this.transformer = new SnowflakeMapToRecordTransformer(schema);
   }
 
diff --git a/src/test/java/io/cdap/plugin/snowflake/common/BaseSnowflakeTest.java b/src/test/java/io/cdap/plugin/snowflake/common/BaseSnowflakeTest.java
@@ -17,13 +17,15 @@
 package io.cdap.plugin.snowflake.common;
 
 import io.cdap.cdap.etl.mock.test.HydratorTestBase;
+import io.cdap.cdap.test.TestConfiguration;
 import io.cdap.plugin.snowflake.Constants;
 import io.cdap.plugin.snowflake.common.client.SnowflakeAccessorTest;
 import io.cdap.plugin.snowflake.source.batch.SnowflakeBatchSourceConfig;
 import io.cdap.plugin.snowflake.source.batch.SnowflakeBatchSourceConfigBuilder;
 import net.snowflake.client.jdbc.SnowflakeBasicDataSource;
 import org.junit.Assume;
 import org.junit.BeforeClass;
+import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.internal.AssumptionViolatedException;
 import org.junit.rules.TestName;
@@ -55,6 +57,9 @@ public abstract class BaseSnowflakeTest extends HydratorTestBase {
 
   private static final Logger LOG = LoggerFactory.getLogger(SnowflakeAccessorTest.class);
 
+  @ClassRule
+  public static final TestConfiguration TEST_CONFIG = new TestConfiguration("explore.enabled", false);
+
   protected static final String ACCOUNT_NAME = System.getProperty("snowflake.test.account.name");
   protected static final String DATABASE = System.getProperty("snowflake.test.database");
   protected static final String SCHEMA = System.getProperty("snowflake.test.schema");
diff --git a/src/test/java/io/cdap/plugin/snowflake/common/util/SchemaHelperTest.java b/src/test/java/io/cdap/plugin/snowflake/common/util/SchemaHelperTest.java
@@ -65,22 +65,24 @@ public void testGetSchemaInvalidJson() {
 
   @Test
   public void testGetSchemaFromSnowflakeUnknownType() throws IOException {
+    String importQuery = "SELECT * FROM someTable";
     MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE);
     SnowflakeSourceAccessor snowflakeAccessor = Mockito.mock(SnowflakeSourceAccessor.class);
 
     List<SnowflakeFieldDescriptor> sample = new ArrayList<>();
     sample.add(new SnowflakeFieldDescriptor("field1", -1000, false));
 
-    Mockito.when(snowflakeAccessor.describeQuery(null)).thenReturn(sample);
+    Mockito.when(snowflakeAccessor.describeQuery(importQuery)).thenReturn(sample);
 
-    SchemaHelper.getSchema(snowflakeAccessor, null, collector, null);
+    SchemaHelper.getSchema(snowflakeAccessor, null, collector, importQuery);
 
     ValidationAssertions.assertValidationFailed(
       collector, Collections.singletonList(SnowflakeBatchSourceConfig.PROPERTY_SCHEMA));
   }
 
   @Test
   public void testGetSchemaFromSnowflake() throws IOException {
+    String importQuery = "SELECT * FROM someTable";
     MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE);
     SnowflakeSourceAccessor snowflakeAccessor = Mockito.mock(SnowflakeSourceAccessor.class);
 
@@ -142,9 +144,9 @@ public void testGetSchemaFromSnowflake() throws IOException {
       Schema.Field.of("field134", Schema.nullableOf(Schema.of(Schema.LogicalType.TIMESTAMP_MICROS)))
     );
 
-    Mockito.when(snowflakeAccessor.describeQuery(null)).thenReturn(sample);
+    Mockito.when(snowflakeAccessor.describeQuery(importQuery)).thenReturn(sample);
 
-    Schema actual = SchemaHelper.getSchema(snowflakeAccessor, null, collector, null);
+    Schema actual = SchemaHelper.getSchema(snowflakeAccessor, null, collector, importQuery);
 
     Assert.assertTrue(collector.getValidationFailures().isEmpty());
     Assert.assertEquals(expected, actual);
@@ -182,4 +184,21 @@ public void testGetSchemaWhenMacroIsEnabledSchemaIsNull() {
     Assert.assertNull(actual);
 
   }
+
+  @Test
+  public void testGetSchemaManuallyUpdatedTheSchema() {
+    Schema expected = Schema.recordOf("test",
+                                      Schema.Field.of("test_field", Schema.nullableOf(Schema.of(Schema.Type.LONG)))
+    );
+
+    SnowflakeBatchSourceConfig mockConfig = Mockito.mock(SnowflakeBatchSourceConfig.class);
+    Mockito.when(mockConfig.canConnect()).thenReturn(false);
+    Mockito.when(mockConfig.getSchema()).thenReturn(expected.toString());
+
+    MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE);
+    Schema actual = SchemaHelper.getSchema(mockConfig, collector);
+
+    Assert.assertTrue(collector.getValidationFailures().isEmpty());
+    Assert.assertEquals(expected, actual);
+  }
 }
diff --git a/src/test/java/io/cdap/plugin/snowflake/source/batch/SnowflakeMapToRecordTransformerTest.java b/src/test/java/io/cdap/plugin/snowflake/source/batch/SnowflakeMapToRecordTransformerTest.java
@@ -50,8 +50,8 @@ public void transform() {
     row.put("COLUMN_CHARACTER", "2");
     row.put("COLUMN_STRING", "text_115");
     row.put("COLUMN_TEXT", "text_116");
-    row.put("COLUMN_BINARY", "text_117");
-    row.put("COLUMN_VARBINARY", "text_118");
+    row.put("COLUMN_BINARY", "746578745f313137");
+    row.put("COLUMN_VARBINARY", "746578745f313138");
     row.put("COLUMN_BOOLEAN", "true");
     row.put("COLUMN_DATE", "2019-01-01");
     row.put("COLUMN_DATETIME", "2019-01-01T01:01:01+00:00");

Original file line number	Diff line number	Diff line change
`@@ -87,8 +87,7 @@ public void prepareRun(BatchSourceContext context) {`
`87`	`87`	`@Override`
`88`	`88`	`public void initialize(BatchRuntimeContext context) throws Exception {`
`89`	`89`	`super.initialize(context);`
`90`		`- SnowflakeSourceAccessor snowflakeAccessor = new SnowflakeSourceAccessor(config);`
`91`		`- Schema schema = SchemaHelper.getSchema(snowflakeAccessor, config.getImportQuery());`
	`90`	`+ Schema schema = SchemaHelper.getSchema(config, context.getFailureCollector());`
`92`	`91`	`this.transformer = new SnowflakeMapToRecordTransformer(schema);`
`93`	`92`	`}`
`94`	`93`