Skip to content

Commit 6994e09

Browse files
MrRahulSharmaMrRahulSharma
authored
Backward Compatibility Fix for Oracle Number type without precision and scale. (#365)
1 parent 7a6ca1d commit 6994e09

5 files changed

Lines changed: 149 additions & 79 deletions

File tree

‎oracle-plugin/docs/Oracle-batchsink.md‎

Lines changed: 32 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -60,37 +60,38 @@ will be passed to the JDBC driver as connection arguments for JDBC drivers that
6060
Data Types Mapping
6161
----------
6262

63-
| Oracle Data Type | CDAP Schema Data Type | Comment |
64-
| ------------------------------ | --------------------- | ------------------------------------------------------ |
65-
| VARCHAR2 | string | |
66-
| NVARCHAR2 | string | |
67-
| VARCHAR | string | |
68-
| NUMBER | string | For NUMBER types defined without a precision and scale |
69-
| NUMBER | decimal | For NUMBER types with a defined precision and scale |
70-
| FLOAT | double | |
71-
| LONG | string | |
72-
| DATE | timestamp | |
73-
| BINARY_FLOAT | float | |
74-
| BINARY_DOUBLE | double | |
75-
| TIMESTAMP | timestamp | |
76-
| TIMESTAMP WITH TIME ZONE | string | Timestamp string in the following format: |
77-
| | | "2019-07-15 15:57:46.65 GMT" |
78-
| TIMESTAMP WITH LOCAL TIME ZONE | timestamp | |
79-
| INTERVAL YEAR TO MONTH | string | Oracle's 'INTERVAL YEAR TO MONTH' literal in the |
80-
| | | standard format: "year[-month]" |
81-
| INTERVAL DAY TO SECOND | string | Oracle's 'INTERVAL DAY TO SECOND' literal in the |
82-
| | | standard format: |
83-
| | | "[day] [hour][:minutes][:seconds[.milliseconds]" |
84-
| RAW | bytes | |
85-
| LONG RAW | bytes | |
86-
| ROWID | string | |
87-
| UROWID | string | |
88-
| CHAR | string | |
89-
| NCHAR | string | |
90-
| CLOB | string | |
91-
| NCLOB | string | |
92-
| BLOB | bytes | |
93-
| BFILE | | BFILE data type is not supported for the sink |
63+
| Oracle Data Type | CDAP Schema Data Type | Comment |
64+
| ------------------------------ | --------------------- | -----------------------------------------------------------|
65+
| VARCHAR2 | string | |
66+
| NVARCHAR2 | string | |
67+
| VARCHAR | string | |
68+
| NUMBER | string | For NUMBER types defined without a precision and scale. |
69+
| | | Users can manually set output schema to map it to Decimal. |
70+
| NUMBER | decimal | For NUMBER types defined with a precision and scale. |
71+
| FLOAT | double | |
72+
| LONG | string | |
73+
| DATE | timestamp | |
74+
| BINARY_FLOAT | float | |
75+
| BINARY_DOUBLE | double | |
76+
| TIMESTAMP | timestamp | |
77+
| TIMESTAMP WITH TIME ZONE | string | Timestamp string in the following format: |
78+
| | | "2019-07-15 15:57:46.65 GMT" |
79+
| TIMESTAMP WITH LOCAL TIME ZONE | timestamp | |
80+
| INTERVAL YEAR TO MONTH | string | Oracle's 'INTERVAL YEAR TO MONTH' literal in the |
81+
| | | standard format: "year[-month]" |
82+
| INTERVAL DAY TO SECOND | string | Oracle's 'INTERVAL DAY TO SECOND' literal in the |
83+
| | | standard format: |
84+
| | | "[day] [hour][:minutes][:seconds[.milliseconds]" |
85+
| RAW | bytes | |
86+
| LONG RAW | bytes | |
87+
| ROWID | string | |
88+
| UROWID | string | |
89+
| CHAR | string | |
90+
| NCHAR | string | |
91+
| CLOB | string | |
92+
| NCLOB | string | |
93+
| BLOB | bytes | |
94+
| BFILE | | BFILE data type is not supported for the sink |
9495

9596

9697
Example

‎oracle-plugin/docs/Oracle-batchsource.md‎

Lines changed: 31 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -74,35 +74,37 @@ with the tradeoff of higher memory usage.
7474
Data Types Mapping
7575
----------
7676

77-
| Oracle Data Type | CDAP Schema Data Type | Comment |
78-
| ------------------------------ | --------------------- | ------------------------------------------------------ |
79-
| VARCHAR2 | string | |
80-
| NVARCHAR2 | string | |
81-
| VARCHAR | string | |
82-
| NUMBER | decimal | |
83-
| FLOAT | double | |
84-
| LONG | string | |
85-
| DATE | timestamp | |
86-
| BINARY_FLOAT | float | |
87-
| BINARY_DOUBLE | double | |
88-
| TIMESTAMP | timestamp | |
89-
| TIMESTAMP WITH TIME ZONE | string | |
90-
| TIMESTAMP WITH LOCAL TIME ZONE | timestamp | |
91-
| INTERVAL YEAR TO MONTH | string | |
92-
| INTERVAL DAY TO SECOND | string | |
93-
| RAW | bytes | |
94-
| LONG RAW | bytes | |
95-
| ROWID | string | |
96-
| UROWID | string | |
97-
| CHAR | string | |
98-
| NCHAR | string | |
99-
| CLOB | string | |
100-
| NCLOB | string | |
101-
| BLOB | bytes | |
102-
| BFILE | bytes | BFILE is a data type used to store a locator (link) |
103-
| | | to an external file, which is stored outside of the |
104-
| | | database. Only the locator will be read from an |
105-
| | | Oracle table and not the content of the external file. |
77+
| Oracle Data Type | CDAP Schema Data Type | Comment |
78+
| ------------------------------ | --------------------- | -----------------------------------------------------------|
79+
| VARCHAR2 | string | |
80+
| NVARCHAR2 | string | |
81+
| VARCHAR | string | |
82+
| NUMBER | string | For NUMBER types defined without a precision and scale. |
83+
| | | Users can manually set output schema to map it to Decimal. |
84+
| NUMBER | decimal | For NUMBER types defined with a precision and scale. |
85+
| FLOAT | double | |
86+
| LONG | string | |
87+
| DATE | timestamp | |
88+
| BINARY_FLOAT | float | |
89+
| BINARY_DOUBLE | double | |
90+
| TIMESTAMP | timestamp | |
91+
| TIMESTAMP WITH TIME ZONE | string | |
92+
| TIMESTAMP WITH LOCAL TIME ZONE | timestamp | |
93+
| INTERVAL YEAR TO MONTH | string | |
94+
| INTERVAL DAY TO SECOND | string | |
95+
| RAW | bytes | |
96+
| LONG RAW | bytes | |
97+
| ROWID | string | |
98+
| UROWID | string | |
99+
| CHAR | string | |
100+
| NCHAR | string | |
101+
| CLOB | string | |
102+
| NCLOB | string | |
103+
| BLOB | bytes | |
104+
| BFILE | bytes | BFILE is a data type used to store a locator (link) |
105+
| | | to an external file, which is stored outside of the |
106+
| | | database. Only the locator will be read from an |
107+
| | | Oracle table and not the content of the external file. |
106108

107109

108110
Example

‎oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import io.cdap.cdap.api.annotation.MetadataProperty;
2424
import io.cdap.cdap.api.annotation.Name;
2525
import io.cdap.cdap.api.annotation.Plugin;
26+
import io.cdap.cdap.api.data.schema.Schema;
2627
import io.cdap.cdap.etl.api.FailureCollector;
2728
import io.cdap.cdap.etl.api.batch.BatchSourceContext;
2829
import io.cdap.cdap.etl.api.connector.Connector;
@@ -153,6 +154,21 @@ public void validate(FailureCollector collector) {
153154
public String getTransactionIsolationLevel() {
154155
return connection.getTransactionIsolationLevel();
155156
}
157+
158+
@Override
159+
protected void validateField(FailureCollector collector,
160+
Schema.Field field, Schema actualFieldSchema, Schema expectedFieldSchema) {
161+
// This change is needed to make sure that the pipeline upgrade continues to work post upgrade.
162+
// Since the older handling of the precision less used to convert to the decimal type,
163+
// and the new version would try to convert to the String type. In that case the output schema would
164+
// contain Decimal(38, 0) (or something similar), and the code internally would try to identify
165+
// the schema of the field(without precision and scale) as String.
166+
if (Schema.LogicalType.DECIMAL.equals(expectedFieldSchema.getLogicalType())
167+
&& actualFieldSchema.getType().equals(Schema.Type.STRING)) {
168+
return;
169+
}
170+
super.validateField(collector, field, actualFieldSchema, expectedFieldSchema);
171+
}
156172
}
157173

158174
/**

‎oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -223,8 +223,16 @@ private void handleOracleSpecificType(ResultSet resultSet, StructuredRecord.Buil
223223
recordBuilder.set(field.getName(), resultSet.getDouble(columnIndex));
224224
} else {
225225
if (precision == 0) {
226-
// In case of precision less decimal convert the field to String type
227-
recordBuilder.set(field.getName(), resultSet.getString(columnIndex));
226+
Schema nonNullableSchema = field.getSchema().isNullable() ?
227+
field.getSchema().getNonNullable() : field.getSchema();
228+
if (Schema.LogicalType.DECIMAL.equals(nonNullableSchema.getLogicalType())) {
229+
// Handle the field using the schema set in the output schema
230+
BigDecimal decimal = resultSet.getBigDecimal(columnIndex, getScale(field.getSchema()));
231+
recordBuilder.setDecimal(field.getName(), decimal);
232+
} else {
233+
// In case of Number defined without precision and scale convert to String type
234+
recordBuilder.set(field.getName(), resultSet.getString(columnIndex));
235+
}
228236
} else {
229237
// It's required to pass 'scale' parameter since in the case of Oracle, scale of 'BigDecimal' depends on the
230238
// scale set in the logical schema. For example for value '77.12' if the scale set in the logical schema is

0 commit comments

Comments
 (0)