Show new error message when output schema is not maching input

bajram-adapt · bajram-adapt · commit c4a79967d575 · 2020-10-30T12:29:13.000-05:00
diff --git a/pom.xml b/pom.xml
@@ -22,16 +22,17 @@
 
   <groupId>io.cdap.plugin</groupId>
   <artifactId>dynamic-spark</artifactId>
-  <version>2.2.2</version>
+  <version>2.2.3-SNAPSHOT</version>
 
   <properties>
     <!-- properties for script build step that creates the config files for the artifacts -->
     <data.pipeline.parent>system:cdap-data-pipeline[6.0.0-SNAPSHOT,7.0.0-SNAPSHOT)</data.pipeline.parent>
     <data.stream.parent>system:cdap-data-streams[6.0.0-SNAPSHOT,7.0.0-SNAPSHOT)</data.stream.parent>
 
-    <cdap.version>6.1.3-SNAPSHOT</cdap.version>
+    <cdap.version>6.1.1</cdap.version>
     <spark.version>2.1.3</spark.version>
     <logback.version>1.0.9</logback.version>
+    <hydrator-common.version>2.4.0</hydrator-common.version>
 
     <surefire.redirectTestOutputToFile>true</surefire.redirectTestOutputToFile>
   </properties>
@@ -100,6 +101,11 @@
       <version>4.11</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>io.cdap.plugin</groupId>
+      <artifactId>hydrator-common</artifactId>
+      <version>${hydrator-common.version}</version>
+    </dependency>
     <dependency>
       <groupId>io.cdap.cdap</groupId>
       <artifactId>hydrator-test</artifactId>
diff --git a/src/main/java/io/cdap/plugin/spark/dynamic/ScalaSparkCompute.java b/src/main/java/io/cdap/plugin/spark/dynamic/ScalaSparkCompute.java
@@ -24,6 +24,7 @@
 import io.cdap.cdap.api.data.schema.Schema;
 import io.cdap.cdap.api.plugin.PluginConfig;
 import io.cdap.cdap.api.spark.sql.DataFrames;
+import io.cdap.cdap.etl.api.FailureCollector;
 import io.cdap.cdap.etl.api.PipelineConfigurer;
 import io.cdap.cdap.etl.api.StageConfigurer;
 import io.cdap.cdap.etl.api.batch.SparkCompute;
@@ -98,6 +99,13 @@ public JavaRDD<StructuredRecord> transform(SparkExecutionPluginContext context,
       // If there is no output schema configured, derive it from the DataFrame
       // Otherwise, assume the DataFrame has the correct schema already
       outputSchema = DataFrames.toSchema((DataType) invokeDataFrameMethod(result, "schema"));
+    } else {
+      Schema dataSchema = DataFrames.toSchema((DataType) invokeDataFrameMethod(result, "schema"));
+      if (!dataSchema.isCompatible(outputSchema)) {
+        FailureCollector collector = context.getFailureCollector();
+        collector.addFailure("Schema mismatch.", "Output schema is not matching input schema.");
+        collector.getOrThrowException();
+      }
     }
     //noinspection unchecked
     return ((JavaRDD<Row>) invokeDataFrameMethod(result, "toJavaRDD")).map(new RowToRecord(outputSchema));
