[Spark SQL] NegativeArraySizeException When Parse InternalRow to DTO Field with Type Array[String]

classic Classic list List threaded Threaded
4 messages Options
Reply | Threaded
Open this post in threaded view
|

[Spark SQL] NegativeArraySizeException When Parse InternalRow to DTO Field with Type Array[String]

Proust (Feng Guizhou) [Travel Search & Discovery]
Hi, Spark Users

I ecounter below NegativeArraySizeException when run Spark SQL. The catalyst generated code for "apply2_19" and "apply1_11" is attached and also the related DTO.
Difficult to understand how the problem could happen, please help if any idea.

I can see maybe https://issues.apache.org/jira/browse/SPARK-15062 is similar but my data type is Array[String] and Spark version is 2.1.2 which looks good both.
By increasing memory to 8G one will instead get a NegativeArraySizeException or a segfault. See here for original discussion: http://apache-spark-developers-list ...
issues.apache.org


java.lang.NegativeArraySizeException
  at org.apache.spark.unsafe.types.UTF8String.getBytes(UTF8String.java:229)
  at org.apache.spark.unsafe.types.UTF8String.toString(UTF8String.java:1005)
  at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply1_11$(generated.java:2467)
  at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply2_19$(generated.java:1475)
  at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply(generated.java:3881)
  at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
  at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
  at scala.collection.Iterator$GroupedIterator.takeDestructively(Iterator.scala:1076)
  at scala.collection.Iterator$GroupedIterator.go(Iterator.scala:1091)
  at scala.collection.Iterator$GroupedIterator.fill(Iterator.scala:1128)
  at scala.collection.Iterator$GroupedIterator.hasNext(Iterator.scala:1132)
  at scala.collection.Iterator$class.foreach(Iterator.scala:893)
  at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
  at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:101)
  at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:95)
  at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
  at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
  at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
  at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
  at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
  at org.apache.spark.scheduler.Task.run(Task.scala:99)
  at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  at java.lang.Thread.run(Thread.java:748)


case class VendorItemDto(
rate_category_id: java.lang.Long,
product_id: java.lang.Long,
vendor_item_rate_category_id: String,
vendor_item_id: java.lang.Long,
name: String,
item_name: String,
vendor_item_order: java.lang.Integer,
option_codes: Array[String],
option_names: Array[String],
feature_values_ids: Array[String],
feature_values_names: Array[String],
benefit_policy_type: String,
benefit_discount_rate: java.lang.Double,
benefit_discount_amount: java.lang.Long,
available_stock: java.lang.Integer,
sale_price: java.lang.Double,
original_price: java.lang.Double,
supply_price: java.lang.Double,
period_id_set: Array[String],
use_start_set: Array[String],
use_end_set: Array[String]
)


/* 2430 */   private void apply1_11(InternalRow i) {
/* 2431 */
/* 2432 */
/* 2433 */     boolean isNull222 = MapObjects_loopIsNull379;
/* 2434 */     ArrayData value222 = null;
/* 2435 */
/* 2436 */     if (!MapObjects_loopIsNull379) {
/* 2437 */
/* 2438 */       if (MapObjects_loopValue378.isNullAt(19)) {
/* 2439 */         isNull222 = true;
/* 2440 */       } else {
/* 2441 */         value222 = MapObjects_loopValue378.getArray(19);
/* 2442 */       }
/* 2443 */
/* 2444 */     }
/* 2445 */     ArrayData value221 = null;
/* 2446 */
/* 2447 */     if (!isNull222) {
/* 2448 */
/* 2449 */       java.lang.String[] convertedArray17 = null;
/* 2450 */       int dataLength17 = value222.numElements();
/* 2451 */       convertedArray17 = new java.lang.String[dataLength17];
/* 2452 */
/* 2453 */       int loopIndex17 = 0;
/* 2454 */       while (loopIndex17 < dataLength17) {
/* 2455 */         MapObjects_loopValue390 = (UTF8String) (value222.getUTF8String(loopIndex17));
/* 2456 */         MapObjects_loopIsNull391 = value222.isNullAt(loopIndex17);
/* 2457 */
/* 2458 */
/* 2459 */         boolean isNull223 = true;
/* 2460 */         java.lang.String value223 = null;
/* 2461 */         if (!MapObjects_loopIsNull391) {
/* 2462 */
/* 2463 */           isNull223 = false;
/* 2464 */           if (!isNull223) {
/* 2465 */
/* 2466 */             Object funcResult52 = null;
/* 2467 */             funcResult52 = MapObjects_loopValue390.toString();
/* 2468 */             if (funcResult52 == null) {
/* 2469 */               isNull223 = true;
/* 2470 */             } else {
/* 2471 */               value223 = (java.lang.String) funcResult52;
/* 2472 */             }
/* 2473 */
/* 2474 */           }
/* 2475 */           isNull223 = value223 == null;
/* 2476 */         }
/* 2477 */         if (isNull223) {
/* 2478 */           convertedArray17[loopIndex17] = null;
/* 2479 */         } else {
/* 2480 */           convertedArray17[loopIndex17] = value223;
/* 2481 */         }
/* 2482 */
/* 2483 */         loopIndex17 += 1;
/* 2484 */       }
/* 2485 */
/* 2486 */       value221 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray17);
/* 2487 */     }
/* 2488 */     boolean isNull220 = true;
/* 2489 */     java.lang.String[] value220 = null;
/* 2490 */     if (!isNull222) {
/* 2491 */
/* 2492 */       isNull220 = false;
/* 2493 */       if (!isNull220) {
/* 2494 */
/* 2495 */         Object funcResult53 = null;
/* 2496 */         funcResult53 = value221.array();
/* 2497 */         if (funcResult53 == null) {
/* 2498 */           isNull220 = true;
/* 2499 */         } else {
/* 2500 */           value220 = (java.lang.String[]) funcResult53;
/* 2501 */         }
/* 2502 */
/* 2503 */       }
/* 2504 */       isNull220 = value220 == null;
/* 2505 */     }
/* 2506 */     argValue111 = value220;
/* 2507 */
/* 2508 */   }


/* 933 */   private void apply2_19(InternalRow i) {
/* 934 */
/* 935 */
/* 936 */     resultIsNull30 = false;
/* 937 */     if (!resultIsNull30) {
/* 938 */
/* 939 */       boolean isNull161 = i.isNullAt(36);
/* 940 */       long value161 = isNull161 ? -1L : (i.getLong(36));
/* 941 */       resultIsNull30 = isNull161;
/* 942 */       argValue90 = value161;
/* 943 */     }
/* 944 */
/* 945 */
/* 946 */     final java.lang.Long value160 = resultIsNull30 ? null : new java.lang.Long(argValue90);
/* 947 */     argValue35 = value160;
/* 948 */
/* 949 */
/* 950 */     resultIsNull31 = false;
/* 951 */     if (!resultIsNull31) {
/* 952 */
/* 953 */       boolean isNull163 = i.isNullAt(37);
/* 954 */       long value163 = isNull163 ? -1L : (i.getLong(37));
/* 955 */       resultIsNull31 = isNull163;
/* 956 */       argValue91 = value163;
/* 957 */     }
/* 958 */
/* 959 */
/* 960 */     final java.lang.Long value162 = resultIsNull31 ? null : new java.lang.Long(argValue91);
/* 961 */     argValue36 = value162;
/* 962 */
/* 963 */
/* 964 */     boolean isNull167 = i.isNullAt(35);
/* 965 */     ArrayData value167 = isNull167 ? null : (i.getArray(35));
/* 966 */     boolean isNull166 = isNull167;
/* 967 */     ArrayData value166 = null;
/* 968 */     if (!isNull167) {
/* 969 */
/* 970 */       final int n = value167.numElements();
/* 971 */       final Object[] values = new Object[n];
/* 972 */       for (int j = 0; j < n; j ++) {
/* 973 */         if (value167.isNullAt(j)) {
/* 974 */           values[j] = null;
/* 975 */         } else {
/* 976 */           boolean feNull = false;
/* 977 */           InternalRow fePrim =
/* 978 */           value167.getStruct(j, 21);
/* 979 */
/* 980 */           boolean teNull = feNull;
/* 981 */           InternalRow tePrim = null;
/* 982 */           if (!feNull) {
/* 983 */
/* 984 */             final org.apache.spark.sql.catalyst.expressions.GenericInternalRow result = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(21);
/* 985 */             final InternalRow tmpRow = fePrim;
/* 986 */
/* 987 */             boolean ffn = tmpRow.isNullAt(0);
/* 988 */             if (ffn) {
/* 989 */               result.setNullAt(0);
/* 990 */             } else {
/* 991 */               long ffp =
/* 992 */               tmpRow.getLong(0);
/* 993 */
/* 994 */               boolean tfn = ffn;
/* 995 */               long tfp = -1L;
/* 996 */               if (!ffn) {
/* 997 */                 tfp = ffp;
/* 998 */               }
/* 999 */
/* 1000 */               if (tfn) {
/* 1001 */                 result.setNullAt(0);
/* 1002 */               } else {
/* 1003 */                 result.setLong(0, tfp);
/* 1004 */               }
/* 1005 */             }
/* 1006 */
/* 1007 */
/* 1008 */             boolean ffn1 = tmpRow.isNullAt(1);
/* 1009 */             if (ffn1) {
/* 1010 */               result.setNullAt(1);
/* 1011 */             } else {
/* 1012 */               long ffp1 =
/* 1013 */               tmpRow.getLong(1);
/* 1014 */
/* 1015 */               boolean tfn1 = ffn1;
/* 1016 */               long tfp1 = -1L;
/* 1017 */               if (!ffn1) {
/* 1018 */                 tfp1 = ffp1;
/* 1019 */               }
/* 1020 */
/* 1021 */               if (tfn1) {
/* 1022 */                 result.setNullAt(1);
/* 1023 */               } else {
/* 1024 */                 result.setLong(1, tfp1);
/* 1025 */               }
/* 1026 */             }
/* 1027 */
/* 1028 */
/* 1029 */             boolean ffn2 = tmpRow.isNullAt(2);
/* 1030 */             if (ffn2) {
/* 1031 */               result.setNullAt(2);
/* 1032 */             } else {
/* 1033 */               UTF8String ffp2 =
/* 1034 */               tmpRow.getUTF8String(2);
/* 1035 */
/* 1036 */               boolean tfn2 = ffn2;
/* 1037 */               UTF8String tfp2 = null;
/* 1038 */               if (!ffn2) {
/* 1039 */                 tfp2 = ffp2;
/* 1040 */               }
/* 1041 */
/* 1042 */               if (tfn2) {
/* 1043 */                 result.setNullAt(2);
/* 1044 */               } else {
/* 1045 */                 result.update(2, tfp2.clone());
/* 1046 */               }
/* 1047 */             }
/* 1048 */
/* 1049 */
/* 1050 */             boolean ffn3 = tmpRow.isNullAt(3);
/* 1051 */             if (ffn3) {
/* 1052 */               result.setNullAt(3);
/* 1053 */             } else {
/* 1054 */               long ffp3 =
/* 1055 */               tmpRow.getLong(3);
/* 1056 */
/* 1057 */               boolean tfn3 = ffn3;
/* 1058 */               long tfp3 = -1L;
/* 1059 */               if (!ffn3) {
/* 1060 */                 tfp3 = ffp3;
/* 1061 */               }
/* 1062 */
/* 1063 */               if (tfn3) {
/* 1064 */                 result.setNullAt(3);
/* 1065 */               } else {
/* 1066 */                 result.setLong(3, tfp3);
/* 1067 */               }
/* 1068 */             }
/* 1069 */
/* 1070 */
/* 1071 */             boolean ffn4 = tmpRow.isNullAt(4);
/* 1072 */             if (ffn4) {
/* 1073 */               result.setNullAt(4);
/* 1074 */             } else {
/* 1075 */               UTF8String ffp4 =
/* 1076 */               tmpRow.getUTF8String(4);
/* 1077 */
/* 1078 */               boolean tfn4 = ffn4;
/* 1079 */               UTF8String tfp4 = null;
/* 1080 */               if (!ffn4) {
/* 1081 */                 tfp4 = ffp4;
/* 1082 */               }
/* 1083 */
/* 1084 */               if (tfn4) {
/* 1085 */                 result.setNullAt(4);
/* 1086 */               } else {
/* 1087 */                 result.update(4, tfp4.clone());
/* 1088 */               }
/* 1089 */             }
/* 1090 */
/* 1091 */
/* 1092 */             boolean ffn5 = tmpRow.isNullAt(5);
/* 1093 */             if (ffn5) {
/* 1094 */               result.setNullAt(5);
/* 1095 */             } else {
/* 1096 */               UTF8String ffp5 =
/* 1097 */               tmpRow.getUTF8String(5);
/* 1098 */
/* 1099 */               boolean tfn5 = ffn5;
/* 1100 */               UTF8String tfp5 = null;
/* 1101 */               if (!ffn5) {
/* 1102 */                 tfp5 = ffp5;
/* 1103 */               }
/* 1104 */
/* 1105 */               if (tfn5) {
/* 1106 */                 result.setNullAt(5);
/* 1107 */               } else {
/* 1108 */                 result.update(5, tfp5.clone());
/* 1109 */               }
/* 1110 */             }
/* 1111 */
/* 1112 */
/* 1113 */             boolean ffn6 = tmpRow.isNullAt(6);
/* 1114 */             if (ffn6) {
/* 1115 */               result.setNullAt(6);
/* 1116 */             } else {
/* 1117 */               int ffp6 =
/* 1118 */               tmpRow.getInt(6);
/* 1119 */
/* 1120 */               boolean tfn6 = ffn6;
/* 1121 */               int tfp6 = -1;
/* 1122 */               if (!ffn6) {
/* 1123 */                 tfp6 = ffp6;
/* 1124 */               }
/* 1125 */
/* 1126 */               if (tfn6) {
/* 1127 */                 result.setNullAt(6);
/* 1128 */               } else {
/* 1129 */                 result.setInt(6, tfp6);
/* 1130 */               }
/* 1131 */             }
/* 1132 */
/* 1133 */
/* 1134 */             boolean ffn7 = tmpRow.isNullAt(7);
/* 1135 */             if (ffn7) {
/* 1136 */               result.setNullAt(7);
/* 1137 */             } else {
/* 1138 */               ArrayData ffp7 =
/* 1139 */               tmpRow.getArray(7);
/* 1140 */
/* 1141 */               boolean tfn7 = ffn7;
/* 1142 */               ArrayData tfp7 = null;
/* 1143 */               if (!ffn7) {
/* 1144 */                 tfp7 = ffp7;
/* 1145 */               }
/* 1146 */
/* 1147 */               if (tfn7) {
/* 1148 */                 result.setNullAt(7);
/* 1149 */               } else {
/* 1150 */                 result.update(7, tfp7);
/* 1151 */               }
/* 1152 */             }
/* 1153 */
/* 1154 */
/* 1155 */             boolean ffn8 = tmpRow.isNullAt(8);
/* 1156 */             if (ffn8) {
/* 1157 */               result.setNullAt(8);
/* 1158 */             } else {
/* 1159 */               ArrayData ffp8 =
/* 1160 */               tmpRow.getArray(8);
/* 1161 */
/* 1162 */               boolean tfn8 = ffn8;
/* 1163 */               ArrayData tfp8 = null;
/* 1164 */               if (!ffn8) {
/* 1165 */                 tfp8 = ffp8;
/* 1166 */               }
/* 1167 */
/* 1168 */               if (tfn8) {
/* 1169 */                 result.setNullAt(8);
/* 1170 */               } else {
/* 1171 */                 result.update(8, tfp8);
/* 1172 */               }
/* 1173 */             }
/* 1174 */
/* 1175 */
/* 1176 */             boolean ffn9 = tmpRow.isNullAt(9);
/* 1177 */             if (ffn9) {
/* 1178 */               result.setNullAt(9);
/* 1179 */             } else {
/* 1180 */               ArrayData ffp9 =
/* 1181 */               tmpRow.getArray(9);
/* 1182 */
/* 1183 */               boolean tfn9 = ffn9;
/* 1184 */               ArrayData tfp9 = null;
/* 1185 */               if (!ffn9) {
/* 1186 */                 tfp9 = ffp9;
/* 1187 */               }
/* 1188 */
/* 1189 */               if (tfn9) {
/* 1190 */                 result.setNullAt(9);
/* 1191 */               } else {
/* 1192 */                 result.update(9, tfp9);
/* 1193 */               }
/* 1194 */             }
/* 1195 */
/* 1196 */
/* 1197 */             boolean ffn10 = tmpRow.isNullAt(10);
/* 1198 */             if (ffn10) {
/* 1199 */               result.setNullAt(10);
/* 1200 */             } else {
/* 1201 */               ArrayData ffp10 =
/* 1202 */               tmpRow.getArray(10);
/* 1203 */
/* 1204 */               boolean tfn10 = ffn10;
/* 1205 */               ArrayData tfp10 = null;
/* 1206 */               if (!ffn10) {
/* 1207 */                 tfp10 = ffp10;
/* 1208 */               }
/* 1209 */
/* 1210 */               if (tfn10) {
/* 1211 */                 result.setNullAt(10);
/* 1212 */               } else {
/* 1213 */                 result.update(10, tfp10);
/* 1214 */               }
/* 1215 */             }
/* 1216 */
/* 1217 */
/* 1218 */             boolean ffn11 = tmpRow.isNullAt(11);
/* 1219 */             if (ffn11) {
/* 1220 */               result.setNullAt(11);
/* 1221 */             } else {
/* 1222 */               UTF8String ffp11 =
/* 1223 */               tmpRow.getUTF8String(11);
/* 1224 */
/* 1225 */               boolean tfn11 = ffn11;
/* 1226 */               UTF8String tfp11 = null;
/* 1227 */               if (!ffn11) {
/* 1228 */                 tfp11 = ffp11;
/* 1229 */               }
/* 1230 */
/* 1231 */               if (tfn11) {
/* 1232 */                 result.setNullAt(11);
/* 1233 */               } else {
/* 1234 */                 result.update(11, tfp11.clone());
/* 1235 */               }
/* 1236 */             }
/* 1237 */
/* 1238 */
/* 1239 */             boolean ffn12 = tmpRow.isNullAt(12);
/* 1240 */             if (ffn12) {
/* 1241 */               result.setNullAt(12);
/* 1242 */             } else {
/* 1243 */               double ffp12 =
/* 1244 */               tmpRow.getDouble(12);
/* 1245 */
/* 1246 */               boolean tfn12 = ffn12;
/* 1247 */               double tfp12 = -1.0;
/* 1248 */               if (!ffn12) {
/* 1249 */                 tfp12 = ffp12;
/* 1250 */               }
/* 1251 */
/* 1252 */               if (tfn12) {
/* 1253 */                 result.setNullAt(12);
/* 1254 */               } else {
/* 1255 */                 result.setDouble(12, tfp12);
/* 1256 */               }
/* 1257 */             }
/* 1258 */
/* 1259 */
/* 1260 */             boolean ffn13 = tmpRow.isNullAt(13);
/* 1261 */             if (ffn13) {
/* 1262 */               result.setNullAt(13);
/* 1263 */             } else {
/* 1264 */               long ffp13 =
/* 1265 */               tmpRow.getLong(13);
/* 1266 */
/* 1267 */               boolean tfn13 = ffn13;
/* 1268 */               long tfp13 = -1L;
/* 1269 */               if (!ffn13) {
/* 1270 */                 tfp13 = ffp13;
/* 1271 */               }
/* 1272 */
/* 1273 */               if (tfn13) {
/* 1274 */                 result.setNullAt(13);
/* 1275 */               } else {
/* 1276 */                 result.setLong(13, tfp13);
/* 1277 */               }
/* 1278 */             }
/* 1279 */
/* 1280 */
/* 1281 */             boolean ffn14 = tmpRow.isNullAt(14);
/* 1282 */             if (ffn14) {
/* 1283 */               result.setNullAt(14);
/* 1284 */             } else {
/* 1285 */               int ffp14 =
/* 1286 */               tmpRow.getInt(14);
/* 1287 */
/* 1288 */               boolean tfn14 = ffn14;
/* 1289 */               int tfp14 = -1;
/* 1290 */               if (!ffn14) {
/* 1291 */                 tfp14 = ffp14;
/* 1292 */               }
/* 1293 */
/* 1294 */               if (tfn14) {
/* 1295 */                 result.setNullAt(14);
/* 1296 */               } else {
/* 1297 */                 result.setInt(14, tfp14);
/* 1298 */               }
/* 1299 */             }
/* 1300 */
/* 1301 */
/* 1302 */             boolean ffn15 = tmpRow.isNullAt(15);
/* 1303 */             if (ffn15) {
/* 1304 */               result.setNullAt(15);
/* 1305 */             } else {
/* 1306 */               double ffp15 =
/* 1307 */               tmpRow.getDouble(15);
/* 1308 */
/* 1309 */               boolean tfn15 = ffn15;
/* 1310 */               double tfp15 = -1.0;
/* 1311 */               if (!ffn15) {
/* 1312 */                 tfp15 = ffp15;
/* 1313 */               }
/* 1314 */
/* 1315 */               if (tfn15) {
/* 1316 */                 result.setNullAt(15);
/* 1317 */               } else {
/* 1318 */                 result.setDouble(15, tfp15);
/* 1319 */               }
/* 1320 */             }
/* 1321 */
/* 1322 */
/* 1323 */             boolean ffn16 = tmpRow.isNullAt(16);
/* 1324 */             if (ffn16) {
/* 1325 */               result.setNullAt(16);
/* 1326 */             } else {
/* 1327 */               double ffp16 =
/* 1328 */               tmpRow.getDouble(16);
/* 1329 */
/* 1330 */               boolean tfn16 = ffn16;
/* 1331 */               double tfp16 = -1.0;
/* 1332 */               if (!ffn16) {
/* 1333 */                 tfp16 = ffp16;
/* 1334 */               }
/* 1335 */
/* 1336 */               if (tfn16) {
/* 1337 */                 result.setNullAt(16);
/* 1338 */               } else {
/* 1339 */                 result.setDouble(16, tfp16);
/* 1340 */               }
/* 1341 */             }
/* 1342 */
/* 1343 */
/* 1344 */             boolean ffn17 = tmpRow.isNullAt(17);
/* 1345 */             if (ffn17) {
/* 1346 */               result.setNullAt(17);
/* 1347 */             } else {
/* 1348 */               double ffp17 =
/* 1349 */               tmpRow.getDouble(17);
/* 1350 */
/* 1351 */               boolean tfn17 = ffn17;
/* 1352 */               double tfp17 = -1.0;
/* 1353 */               if (!ffn17) {
/* 1354 */                 tfp17 = ffp17;
/* 1355 */               }
/* 1356 */
/* 1357 */               if (tfn17) {
/* 1358 */                 result.setNullAt(17);
/* 1359 */               } else {
/* 1360 */                 result.setDouble(17, tfp17);
/* 1361 */               }
/* 1362 */             }
/* 1363 */
/* 1364 */
/* 1365 */             boolean ffn18 = tmpRow.isNullAt(18);
/* 1366 */             if (ffn18) {
/* 1367 */               result.setNullAt(18);
/* 1368 */             } else {
/* 1369 */               ArrayData ffp18 =
/* 1370 */               tmpRow.getArray(18);
/* 1371 */
/* 1372 */               boolean tfn18 = ffn18;
/* 1373 */               ArrayData tfp18 = null;
/* 1374 */               if (!ffn18) {
/* 1375 */                 tfp18 = ffp18;
/* 1376 */               }
/* 1377 */
/* 1378 */               if (tfn18) {
/* 1379 */                 result.setNullAt(18);
/* 1380 */               } else {
/* 1381 */                 result.update(18, tfp18);
/* 1382 */               }
/* 1383 */             }
/* 1384 */
/* 1385 */
/* 1386 */             boolean ffn19 = tmpRow.isNullAt(19);
/* 1387 */             if (ffn19) {
/* 1388 */               result.setNullAt(19);
/* 1389 */             } else {
/* 1390 */               ArrayData ffp19 =
/* 1391 */               tmpRow.getArray(19);
/* 1392 */
/* 1393 */               boolean tfn19 = ffn19;
/* 1394 */               ArrayData tfp19 = null;
/* 1395 */               if (!ffn19) {
/* 1396 */                 tfp19 = ffp19;
/* 1397 */               }
/* 1398 */
/* 1399 */               if (tfn19) {
/* 1400 */                 result.setNullAt(19);
/* 1401 */               } else {
/* 1402 */                 result.update(19, tfp19);
/* 1403 */               }
/* 1404 */             }
/* 1405 */
/* 1406 */
/* 1407 */             boolean ffn20 = tmpRow.isNullAt(20);
/* 1408 */             if (ffn20) {
/* 1409 */               result.setNullAt(20);
/* 1410 */             } else {
/* 1411 */               ArrayData ffp20 =
/* 1412 */               tmpRow.getArray(20);
/* 1413 */
/* 1414 */               boolean tfn20 = ffn20;
/* 1415 */               ArrayData tfp20 = null;
/* 1416 */               if (!ffn20) {
/* 1417 */                 tfp20 = ffp20;
/* 1418 */               }
/* 1419 */
/* 1420 */               if (tfn20) {
/* 1421 */                 result.setNullAt(20);
/* 1422 */               } else {
/* 1423 */                 result.update(20, tfp20);
/* 1424 */               }
/* 1425 */             }
/* 1426 */
/* 1427 */             tePrim = result.copy();
/* 1428 */
/* 1429 */           }
/* 1430 */
/* 1431 */           if (teNull) {
/* 1432 */             values[j] = null;
/* 1433 */           } else {
/* 1434 */             values[j] = tePrim;
/* 1435 */           }
/* 1436 */         }
/* 1437 */       }
/* 1438 */       value166 = new org.apache.spark.sql.catalyst.util.GenericArrayData(values);
/* 1439 */
/* 1440 */     }
/* 1441 */     ArrayData value165 = null;
/* 1442 */
/* 1443 */     if (!isNull166) {
/* 1444 */
/* 1445 */       indexer.dto.ticket.VendorItemDto[] convertedArray19 = null;
/* 1446 */       int dataLength19 = value166.numElements();
/* 1447 */       convertedArray19 = new indexer.dto.ticket.VendorItemDto[dataLength19];
/* 1448 */
/* 1449 */       int loopIndex19 = 0;
/* 1450 */       while (loopIndex19 < dataLength19) {
/* 1451 */         MapObjects_loopValue378 = (InternalRow) (value166.getStruct(loopIndex19, 21));
/* 1452 */         MapObjects_loopIsNull379 = value166.isNullAt(loopIndex19);
/* 1453 */
/* 1454 */
/* 1455 */         boolean isNull168 = false;
/* 1456 */         indexer.dto.ticket.VendorItemDto value168 = null;
/* 1457 */         if (!false && MapObjects_loopIsNull379) {
/* 1458 */
/* 1459 */           final indexer.dto.ticket.VendorItemDto value170 = null;
/* 1460 */           isNull168 = true;
/* 1461 */           value168 = value170;
/* 1462 */         } else {
/* 1463 */
/* 1464 */           apply1_0(i);
/* 1465 */           apply1_1(i);
/* 1466 */           apply1_2(i);
/* 1467 */           apply1_3(i);
/* 1468 */           apply1_4(i);
/* 1469 */           apply1_5(i);
/* 1470 */           apply1_6(i);
/* 1471 */           apply1_7(i);
/* 1472 */           apply1_8(i);
/* 1473 */           apply1_9(i);
/* 1474 */           apply1_10(i);
/* 1475 */           apply1_11(i);
/* 1476 */           apply1_12(i);
/* 1477 */
/* 1478 */           final indexer.dto.ticket.VendorItemDto value171 = false ? null : new indexer.dto.ticket.VendorItemDto(argValue92, argValue93, argValue94, argValue95, argValue96, argValue97, argValue98, argValue99, argValue100, argValue101, argValue102, argValue103, argValue104, argValue105, argValue106, argValue107, argValue108, argValue109, argValue110, argValue111, argValue112);
/* 1479 */           isNull168 = false;
/* 1480 */           value168 = value171;
/* 1481 */         }
/* 1482 */         if (isNull168) {
/* 1483 */           convertedArray19[loopIndex19] = null;
/* 1484 */         } else {
/* 1485 */           convertedArray19[loopIndex19] = value168;
/* 1486 */         }
/* 1487 */
/* 1488 */         loopIndex19 += 1;
/* 1489 */       }
/* 1490 */
/* 1491 */       value165 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray19);
/* 1492 */     }
/* 1493 */     boolean isNull164 = true;
/* 1494 */     indexer.dto.ticket.VendorItemDto[] value164 = null;
/* 1495 */     if (!isNull166) {
/* 1496 */
/* 1497 */       isNull164 = false;
/* 1498 */       if (!isNull164) {
/* 1499 */
/* 1500 */         Object funcResult56 = null;
/* 1501 */         funcResult56 = value165.array();
/* 1502 */         if (funcResult56 == null) {
/* 1503 */           isNull164 = true;
/* 1504 */         } else {
/* 1505 */           value164 = (indexer.dto.ticket.VendorItemDto[]) funcResult56;
/* 1506 */         }
/* 1507 */
/* 1508 */       }
/* 1509 */       isNull164 = value164 == null;
/* 1510 */     }
/* 1511 */     argValue37 = value164;
/* 1512 */
/* 1513 */   }


Thanks a lot
Proust



地址: 上海市浦东新区金科路2889弄长泰广场C座7楼 
Address: 7th floor, No#3, Jinke road 2889, Pudong district, Shanghai, China.
Mobile: +86 13621672634
Reply | Threaded
Open this post in threaded view
|

Re: [Spark SQL] NegativeArraySizeException When Parse InternalRow to DTO Field with Type Array[String]

Sandeep Patra
This might be due to the serializer being used.

On Sun, Feb 23, 2020 at 1:39 PM Proust (Feng Guizhou) [Travel Search & Discovery] <[hidden email]> wrote:
Hi, Spark Users

I ecounter below NegativeArraySizeException when run Spark SQL. The catalyst generated code for "apply2_19" and "apply1_11" is attached and also the related DTO.
Difficult to understand how the problem could happen, please help if any idea.

I can see maybe https://issues.apache.org/jira/browse/SPARK-15062 is similar but my data type is Array[String] and Spark version is 2.1.2 which looks good both.
By increasing memory to 8G one will instead get a NegativeArraySizeException or a segfault. See here for original discussion: http://apache-spark-developers-list ...


java.lang.NegativeArraySizeException
  at org.apache.spark.unsafe.types.UTF8String.getBytes(UTF8String.java:229)
  at org.apache.spark.unsafe.types.UTF8String.toString(UTF8String.java:1005)
  at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply1_11$(generated.java:2467)
  at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply2_19$(generated.java:1475)
  at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply(generated.java:3881)
  at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
  at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
  at scala.collection.Iterator$GroupedIterator.takeDestructively(Iterator.scala:1076)
  at scala.collection.Iterator$GroupedIterator.go(Iterator.scala:1091)
  at scala.collection.Iterator$GroupedIterator.fill(Iterator.scala:1128)
  at scala.collection.Iterator$GroupedIterator.hasNext(Iterator.scala:1132)
  at scala.collection.Iterator$class.foreach(Iterator.scala:893)
  at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
  at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:101)
  at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:95)
  at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
  at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
  at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
  at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
  at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
  at org.apache.spark.scheduler.Task.run(Task.scala:99)
  at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  at java.lang.Thread.run(Thread.java:748)


case class VendorItemDto(
rate_category_id: java.lang.Long,
product_id: java.lang.Long,
vendor_item_rate_category_id: String,
vendor_item_id: java.lang.Long,
name: String,
item_name: String,
vendor_item_order: java.lang.Integer,
option_codes: Array[String],
option_names: Array[String],
feature_values_ids: Array[String],
feature_values_names: Array[String],
benefit_policy_type: String,
benefit_discount_rate: java.lang.Double,
benefit_discount_amount: java.lang.Long,
available_stock: java.lang.Integer,
sale_price: java.lang.Double,
original_price: java.lang.Double,
supply_price: java.lang.Double,
period_id_set: Array[String],
use_start_set: Array[String],
use_end_set: Array[String]
)


/* 2430 */   private void apply1_11(InternalRow i) {
/* 2431 */
/* 2432 */
/* 2433 */     boolean isNull222 = MapObjects_loopIsNull379;
/* 2434 */     ArrayData value222 = null;
/* 2435 */
/* 2436 */     if (!MapObjects_loopIsNull379) {
/* 2437 */
/* 2438 */       if (MapObjects_loopValue378.isNullAt(19)) {
/* 2439 */         isNull222 = true;
/* 2440 */       } else {
/* 2441 */         value222 = MapObjects_loopValue378.getArray(19);
/* 2442 */       }
/* 2443 */
/* 2444 */     }
/* 2445 */     ArrayData value221 = null;
/* 2446 */
/* 2447 */     if (!isNull222) {
/* 2448 */
/* 2449 */       java.lang.String[] convertedArray17 = null;
/* 2450 */       int dataLength17 = value222.numElements();
/* 2451 */       convertedArray17 = new java.lang.String[dataLength17];
/* 2452 */
/* 2453 */       int loopIndex17 = 0;
/* 2454 */       while (loopIndex17 &lt; dataLength17) {
/* 2455 */         MapObjects_loopValue390 = (UTF8String) (value222.getUTF8String(loopIndex17));
/* 2456 */         MapObjects_loopIsNull391 = value222.isNullAt(loopIndex17);
/* 2457 */
/* 2458 */
/* 2459 */         boolean isNull223 = true;
/* 2460 */         java.lang.String value223 = null;
/* 2461 */         if (!MapObjects_loopIsNull391) {
/* 2462 */
/* 2463 */           isNull223 = false;
/* 2464 */           if (!isNull223) {
/* 2465 */
/* 2466 */             Object funcResult52 = null;
/* 2467 */             funcResult52 = MapObjects_loopValue390.toString();
/* 2468 */             if (funcResult52 == null) {
/* 2469 */               isNull223 = true;
/* 2470 */             } else {
/* 2471 */               value223 = (java.lang.String) funcResult52;
/* 2472 */             }
/* 2473 */
/* 2474 */           }
/* 2475 */           isNull223 = value223 == null;
/* 2476 */         }
/* 2477 */         if (isNull223) {
/* 2478 */           convertedArray17[loopIndex17] = null;
/* 2479 */         } else {
/* 2480 */           convertedArray17[loopIndex17] = value223;
/* 2481 */         }
/* 2482 */
/* 2483 */         loopIndex17 += 1;
/* 2484 */       }
/* 2485 */
/* 2486 */       value221 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray17);
/* 2487 */     }
/* 2488 */     boolean isNull220 = true;
/* 2489 */     java.lang.String[] value220 = null;
/* 2490 */     if (!isNull222) {
/* 2491 */
/* 2492 */       isNull220 = false;
/* 2493 */       if (!isNull220) {
/* 2494 */
/* 2495 */         Object funcResult53 = null;
/* 2496 */         funcResult53 = value221.array();
/* 2497 */         if (funcResult53 == null) {
/* 2498 */           isNull220 = true;
/* 2499 */         } else {
/* 2500 */           value220 = (java.lang.String[]) funcResult53;
/* 2501 */         }
/* 2502 */
/* 2503 */       }
/* 2504 */       isNull220 = value220 == null;
/* 2505 */     }
/* 2506 */     argValue111 = value220;
/* 2507 */
/* 2508 */   }


/* 933 */   private void apply2_19(InternalRow i) {
/* 934 */
/* 935 */
/* 936 */     resultIsNull30 = false;
/* 937 */     if (!resultIsNull30) {
/* 938 */
/* 939 */       boolean isNull161 = i.isNullAt(36);
/* 940 */       long value161 = isNull161 ? -1L : (i.getLong(36));
/* 941 */       resultIsNull30 = isNull161;
/* 942 */       argValue90 = value161;
/* 943 */     }
/* 944 */
/* 945 */
/* 946 */     final java.lang.Long value160 = resultIsNull30 ? null : new java.lang.Long(argValue90);
/* 947 */     argValue35 = value160;
/* 948 */
/* 949 */
/* 950 */     resultIsNull31 = false;
/* 951 */     if (!resultIsNull31) {
/* 952 */
/* 953 */       boolean isNull163 = i.isNullAt(37);
/* 954 */       long value163 = isNull163 ? -1L : (i.getLong(37));
/* 955 */       resultIsNull31 = isNull163;
/* 956 */       argValue91 = value163;
/* 957 */     }
/* 958 */
/* 959 */
/* 960 */     final java.lang.Long value162 = resultIsNull31 ? null : new java.lang.Long(argValue91);
/* 961 */     argValue36 = value162;
/* 962 */
/* 963 */
/* 964 */     boolean isNull167 = i.isNullAt(35);
/* 965 */     ArrayData value167 = isNull167 ? null : (i.getArray(35));
/* 966 */     boolean isNull166 = isNull167;
/* 967 */     ArrayData value166 = null;
/* 968 */     if (!isNull167) {
/* 969 */
/* 970 */       final int n = value167.numElements();
/* 971 */       final Object[] values = new Object[n];
/* 972 */       for (int j = 0; j &lt; n; j ++) {
/* 973 */         if (value167.isNullAt(j)) {
/* 974 */           values[j] = null;
/* 975 */         } else {
/* 976 */           boolean feNull = false;
/* 977 */           InternalRow fePrim =
/* 978 */           value167.getStruct(j, 21);
/* 979 */
/* 980 */           boolean teNull = feNull;
/* 981 */           InternalRow tePrim = null;
/* 982 */           if (!feNull) {
/* 983 */
/* 984 */             final org.apache.spark.sql.catalyst.expressions.GenericInternalRow result = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(21);
/* 985 */             final InternalRow tmpRow = fePrim;
/* 986 */
/* 987 */             boolean ffn = tmpRow.isNullAt(0);
/* 988 */             if (ffn) {
/* 989 */               result.setNullAt(0);
/* 990 */             } else {
/* 991 */               long ffp =
/* 992 */               tmpRow.getLong(0);
/* 993 */
/* 994 */               boolean tfn = ffn;
/* 995 */               long tfp = -1L;
/* 996 */               if (!ffn) {
/* 997 */                 tfp = ffp;
/* 998 */               }
/* 999 */
/* 1000 */               if (tfn) {
/* 1001 */                 result.setNullAt(0);
/* 1002 */               } else {
/* 1003 */                 result.setLong(0, tfp);
/* 1004 */               }
/* 1005 */             }
/* 1006 */
/* 1007 */
/* 1008 */             boolean ffn1 = tmpRow.isNullAt(1);
/* 1009 */             if (ffn1) {
/* 1010 */               result.setNullAt(1);
/* 1011 */             } else {
/* 1012 */               long ffp1 =
/* 1013 */               tmpRow.getLong(1);
/* 1014 */
/* 1015 */               boolean tfn1 = ffn1;
/* 1016 */               long tfp1 = -1L;
/* 1017 */               if (!ffn1) {
/* 1018 */                 tfp1 = ffp1;
/* 1019 */               }
/* 1020 */
/* 1021 */               if (tfn1) {
/* 1022 */                 result.setNullAt(1);
/* 1023 */               } else {
/* 1024 */                 result.setLong(1, tfp1);
/* 1025 */               }
/* 1026 */             }
/* 1027 */
/* 1028 */
/* 1029 */             boolean ffn2 = tmpRow.isNullAt(2);
/* 1030 */             if (ffn2) {
/* 1031 */               result.setNullAt(2);
/* 1032 */             } else {
/* 1033 */               UTF8String ffp2 =
/* 1034 */               tmpRow.getUTF8String(2);
/* 1035 */
/* 1036 */               boolean tfn2 = ffn2;
/* 1037 */               UTF8String tfp2 = null;
/* 1038 */               if (!ffn2) {
/* 1039 */                 tfp2 = ffp2;
/* 1040 */               }
/* 1041 */
/* 1042 */               if (tfn2) {
/* 1043 */                 result.setNullAt(2);
/* 1044 */               } else {
/* 1045 */                 result.update(2, tfp2.clone());
/* 1046 */               }
/* 1047 */             }
/* 1048 */
/* 1049 */
/* 1050 */             boolean ffn3 = tmpRow.isNullAt(3);
/* 1051 */             if (ffn3) {
/* 1052 */               result.setNullAt(3);
/* 1053 */             } else {
/* 1054 */               long ffp3 =
/* 1055 */               tmpRow.getLong(3);
/* 1056 */
/* 1057 */               boolean tfn3 = ffn3;
/* 1058 */               long tfp3 = -1L;
/* 1059 */               if (!ffn3) {
/* 1060 */                 tfp3 = ffp3;
/* 1061 */               }
/* 1062 */
/* 1063 */               if (tfn3) {
/* 1064 */                 result.setNullAt(3);
/* 1065 */               } else {
/* 1066 */                 result.setLong(3, tfp3);
/* 1067 */               }
/* 1068 */             }
/* 1069 */
/* 1070 */
/* 1071 */             boolean ffn4 = tmpRow.isNullAt(4);
/* 1072 */             if (ffn4) {
/* 1073 */               result.setNullAt(4);
/* 1074 */             } else {
/* 1075 */               UTF8String ffp4 =
/* 1076 */               tmpRow.getUTF8String(4);
/* 1077 */
/* 1078 */               boolean tfn4 = ffn4;
/* 1079 */               UTF8String tfp4 = null;
/* 1080 */               if (!ffn4) {
/* 1081 */                 tfp4 = ffp4;
/* 1082 */               }
/* 1083 */
/* 1084 */               if (tfn4) {
/* 1085 */                 result.setNullAt(4);
/* 1086 */               } else {
/* 1087 */                 result.update(4, tfp4.clone());
/* 1088 */               }
/* 1089 */             }
/* 1090 */
/* 1091 */
/* 1092 */             boolean ffn5 = tmpRow.isNullAt(5);
/* 1093 */             if (ffn5) {
/* 1094 */               result.setNullAt(5);
/* 1095 */             } else {
/* 1096 */               UTF8String ffp5 =
/* 1097 */               tmpRow.getUTF8String(5);
/* 1098 */
/* 1099 */               boolean tfn5 = ffn5;
/* 1100 */               UTF8String tfp5 = null;
/* 1101 */               if (!ffn5) {
/* 1102 */                 tfp5 = ffp5;
/* 1103 */               }
/* 1104 */
/* 1105 */               if (tfn5) {
/* 1106 */                 result.setNullAt(5);
/* 1107 */               } else {
/* 1108 */                 result.update(5, tfp5.clone());
/* 1109 */               }
/* 1110 */             }
/* 1111 */
/* 1112 */
/* 1113 */             boolean ffn6 = tmpRow.isNullAt(6);
/* 1114 */             if (ffn6) {
/* 1115 */               result.setNullAt(6);
/* 1116 */             } else {
/* 1117 */               int ffp6 =
/* 1118 */               tmpRow.getInt(6);
/* 1119 */
/* 1120 */               boolean tfn6 = ffn6;
/* 1121 */               int tfp6 = -1;
/* 1122 */               if (!ffn6) {
/* 1123 */                 tfp6 = ffp6;
/* 1124 */               }
/* 1125 */
/* 1126 */               if (tfn6) {
/* 1127 */                 result.setNullAt(6);
/* 1128 */               } else {
/* 1129 */                 result.setInt(6, tfp6);
/* 1130 */               }
/* 1131 */             }
/* 1132 */
/* 1133 */
/* 1134 */             boolean ffn7 = tmpRow.isNullAt(7);
/* 1135 */             if (ffn7) {
/* 1136 */               result.setNullAt(7);
/* 1137 */             } else {
/* 1138 */               ArrayData ffp7 =
/* 1139 */               tmpRow.getArray(7);
/* 1140 */
/* 1141 */               boolean tfn7 = ffn7;
/* 1142 */               ArrayData tfp7 = null;
/* 1143 */               if (!ffn7) {
/* 1144 */                 tfp7 = ffp7;
/* 1145 */               }
/* 1146 */
/* 1147 */               if (tfn7) {
/* 1148 */                 result.setNullAt(7);
/* 1149 */               } else {
/* 1150 */                 result.update(7, tfp7);
/* 1151 */               }
/* 1152 */             }
/* 1153 */
/* 1154 */
/* 1155 */             boolean ffn8 = tmpRow.isNullAt(8);
/* 1156 */             if (ffn8) {
/* 1157 */               result.setNullAt(8);
/* 1158 */             } else {
/* 1159 */               ArrayData ffp8 =
/* 1160 */               tmpRow.getArray(8);
/* 1161 */
/* 1162 */               boolean tfn8 = ffn8;
/* 1163 */               ArrayData tfp8 = null;
/* 1164 */               if (!ffn8) {
/* 1165 */                 tfp8 = ffp8;
/* 1166 */               }
/* 1167 */
/* 1168 */               if (tfn8) {
/* 1169 */                 result.setNullAt(8);
/* 1170 */               } else {
/* 1171 */                 result.update(8, tfp8);
/* 1172 */               }
/* 1173 */             }
/* 1174 */
/* 1175 */
/* 1176 */             boolean ffn9 = tmpRow.isNullAt(9);
/* 1177 */             if (ffn9) {
/* 1178 */               result.setNullAt(9);
/* 1179 */             } else {
/* 1180 */               ArrayData ffp9 =
/* 1181 */               tmpRow.getArray(9);
/* 1182 */
/* 1183 */               boolean tfn9 = ffn9;
/* 1184 */               ArrayData tfp9 = null;
/* 1185 */               if (!ffn9) {
/* 1186 */                 tfp9 = ffp9;
/* 1187 */               }
/* 1188 */
/* 1189 */               if (tfn9) {
/* 1190 */                 result.setNullAt(9);
/* 1191 */               } else {
/* 1192 */                 result.update(9, tfp9);
/* 1193 */               }
/* 1194 */             }
/* 1195 */
/* 1196 */
/* 1197 */             boolean ffn10 = tmpRow.isNullAt(10);
/* 1198 */             if (ffn10) {
/* 1199 */               result.setNullAt(10);
/* 1200 */             } else {
/* 1201 */               ArrayData ffp10 =
/* 1202 */               tmpRow.getArray(10);
/* 1203 */
/* 1204 */               boolean tfn10 = ffn10;
/* 1205 */               ArrayData tfp10 = null;
/* 1206 */               if (!ffn10) {
/* 1207 */                 tfp10 = ffp10;
/* 1208 */               }
/* 1209 */
/* 1210 */               if (tfn10) {
/* 1211 */                 result.setNullAt(10);
/* 1212 */               } else {
/* 1213 */                 result.update(10, tfp10);
/* 1214 */               }
/* 1215 */             }
/* 1216 */
/* 1217 */
/* 1218 */             boolean ffn11 = tmpRow.isNullAt(11);
/* 1219 */             if (ffn11) {
/* 1220 */               result.setNullAt(11);
/* 1221 */             } else {
/* 1222 */               UTF8String ffp11 =
/* 1223 */               tmpRow.getUTF8String(11);
/* 1224 */
/* 1225 */               boolean tfn11 = ffn11;
/* 1226 */               UTF8String tfp11 = null;
/* 1227 */               if (!ffn11) {
/* 1228 */                 tfp11 = ffp11;
/* 1229 */               }
/* 1230 */
/* 1231 */               if (tfn11) {
/* 1232 */                 result.setNullAt(11);
/* 1233 */               } else {
/* 1234 */                 result.update(11, tfp11.clone());
/* 1235 */               }
/* 1236 */             }
/* 1237 */
/* 1238 */
/* 1239 */             boolean ffn12 = tmpRow.isNullAt(12);
/* 1240 */             if (ffn12) {
/* 1241 */               result.setNullAt(12);
/* 1242 */             } else {
/* 1243 */               double ffp12 =
/* 1244 */               tmpRow.getDouble(12);
/* 1245 */
/* 1246 */               boolean tfn12 = ffn12;
/* 1247 */               double tfp12 = -1.0;
/* 1248 */               if (!ffn12) {
/* 1249 */                 tfp12 = ffp12;
/* 1250 */               }
/* 1251 */
/* 1252 */               if (tfn12) {
/* 1253 */                 result.setNullAt(12);
/* 1254 */               } else {
/* 1255 */                 result.setDouble(12, tfp12);
/* 1256 */               }
/* 1257 */             }
/* 1258 */
/* 1259 */
/* 1260 */             boolean ffn13 = tmpRow.isNullAt(13);
/* 1261 */             if (ffn13) {
/* 1262 */               result.setNullAt(13);
/* 1263 */             } else {
/* 1264 */               long ffp13 =
/* 1265 */               tmpRow.getLong(13);
/* 1266 */
/* 1267 */               boolean tfn13 = ffn13;
/* 1268 */               long tfp13 = -1L;
/* 1269 */               if (!ffn13) {
/* 1270 */                 tfp13 = ffp13;
/* 1271 */               }
/* 1272 */
/* 1273 */               if (tfn13) {
/* 1274 */                 result.setNullAt(13);
/* 1275 */               } else {
/* 1276 */                 result.setLong(13, tfp13);
/* 1277 */               }
/* 1278 */             }
/* 1279 */
/* 1280 */
/* 1281 */             boolean ffn14 = tmpRow.isNullAt(14);
/* 1282 */             if (ffn14) {
/* 1283 */               result.setNullAt(14);
/* 1284 */             } else {
/* 1285 */               int ffp14 =
/* 1286 */               tmpRow.getInt(14);
/* 1287 */
/* 1288 */               boolean tfn14 = ffn14;
/* 1289 */               int tfp14 = -1;
/* 1290 */               if (!ffn14) {
/* 1291 */                 tfp14 = ffp14;
/* 1292 */               }
/* 1293 */
/* 1294 */               if (tfn14) {
/* 1295 */                 result.setNullAt(14);
/* 1296 */               } else {
/* 1297 */                 result.setInt(14, tfp14);
/* 1298 */               }
/* 1299 */             }
/* 1300 */
/* 1301 */
/* 1302 */             boolean ffn15 = tmpRow.isNullAt(15);
/* 1303 */             if (ffn15) {
/* 1304 */               result.setNullAt(15);
/* 1305 */             } else {
/* 1306 */               double ffp15 =
/* 1307 */               tmpRow.getDouble(15);
/* 1308 */
/* 1309 */               boolean tfn15 = ffn15;
/* 1310 */               double tfp15 = -1.0;
/* 1311 */               if (!ffn15) {
/* 1312 */                 tfp15 = ffp15;
/* 1313 */               }
/* 1314 */
/* 1315 */               if (tfn15) {
/* 1316 */                 result.setNullAt(15);
/* 1317 */               } else {
/* 1318 */                 result.setDouble(15, tfp15);
/* 1319 */               }
/* 1320 */             }
/* 1321 */
/* 1322 */
/* 1323 */             boolean ffn16 = tmpRow.isNullAt(16);
/* 1324 */             if (ffn16) {
/* 1325 */               result.setNullAt(16);
/* 1326 */             } else {
/* 1327 */               double ffp16 =
/* 1328 */               tmpRow.getDouble(16);
/* 1329 */
/* 1330 */               boolean tfn16 = ffn16;
/* 1331 */               double tfp16 = -1.0;
/* 1332 */               if (!ffn16) {
/* 1333 */                 tfp16 = ffp16;
/* 1334 */               }
/* 1335 */
/* 1336 */               if (tfn16) {
/* 1337 */                 result.setNullAt(16);
/* 1338 */               } else {
/* 1339 */                 result.setDouble(16, tfp16);
/* 1340 */               }
/* 1341 */             }
/* 1342 */
/* 1343 */
/* 1344 */             boolean ffn17 = tmpRow.isNullAt(17);
/* 1345 */             if (ffn17) {
/* 1346 */               result.setNullAt(17);
/* 1347 */             } else {
/* 1348 */               double ffp17 =
/* 1349 */               tmpRow.getDouble(17);
/* 1350 */
/* 1351 */               boolean tfn17 = ffn17;
/* 1352 */               double tfp17 = -1.0;
/* 1353 */               if (!ffn17) {
/* 1354 */                 tfp17 = ffp17;
/* 1355 */               }
/* 1356 */
/* 1357 */               if (tfn17) {
/* 1358 */                 result.setNullAt(17);
/* 1359 */               } else {
/* 1360 */                 result.setDouble(17, tfp17);
/* 1361 */               }
/* 1362 */             }
/* 1363 */
/* 1364 */
/* 1365 */             boolean ffn18 = tmpRow.isNullAt(18);
/* 1366 */             if (ffn18) {
/* 1367 */               result.setNullAt(18);
/* 1368 */             } else {
/* 1369 */               ArrayData ffp18 =
/* 1370 */               tmpRow.getArray(18);
/* 1371 */
/* 1372 */               boolean tfn18 = ffn18;
/* 1373 */               ArrayData tfp18 = null;
/* 1374 */               if (!ffn18) {
/* 1375 */                 tfp18 = ffp18;
/* 1376 */               }
/* 1377 */
/* 1378 */               if (tfn18) {
/* 1379 */                 result.setNullAt(18);
/* 1380 */               } else {
/* 1381 */                 result.update(18, tfp18);
/* 1382 */               }
/* 1383 */             }
/* 1384 */
/* 1385 */
/* 1386 */             boolean ffn19 = tmpRow.isNullAt(19);
/* 1387 */             if (ffn19) {
/* 1388 */               result.setNullAt(19);
/* 1389 */             } else {
/* 1390 */               ArrayData ffp19 =
/* 1391 */               tmpRow.getArray(19);
/* 1392 */
/* 1393 */               boolean tfn19 = ffn19;
/* 1394 */               ArrayData tfp19 = null;
/* 1395 */               if (!ffn19) {
/* 1396 */                 tfp19 = ffp19;
/* 1397 */               }
/* 1398 */
/* 1399 */               if (tfn19) {
/* 1400 */                 result.setNullAt(19);
/* 1401 */               } else {
/* 1402 */                 result.update(19, tfp19);
/* 1403 */               }
/* 1404 */             }
/* 1405 */
/* 1406 */
/* 1407 */             boolean ffn20 = tmpRow.isNullAt(20);
/* 1408 */             if (ffn20) {
/* 1409 */               result.setNullAt(20);
/* 1410 */             } else {
/* 1411 */               ArrayData ffp20 =
/* 1412 */               tmpRow.getArray(20);
/* 1413 */
/* 1414 */               boolean tfn20 = ffn20;
/* 1415 */               ArrayData tfp20 = null;
/* 1416 */               if (!ffn20) {
/* 1417 */                 tfp20 = ffp20;
/* 1418 */               }
/* 1419 */
/* 1420 */               if (tfn20) {
/* 1421 */                 result.setNullAt(20);
/* 1422 */               } else {
/* 1423 */                 result.update(20, tfp20);
/* 1424 */               }
/* 1425 */             }
/* 1426 */
/* 1427 */             tePrim = result.copy();
/* 1428 */
/* 1429 */           }
/* 1430 */
/* 1431 */           if (teNull) {
/* 1432 */             values[j] = null;
/* 1433 */           } else {
/* 1434 */             values[j] = tePrim;
/* 1435 */           }
/* 1436 */         }
/* 1437 */       }
/* 1438 */       value166 = new org.apache.spark.sql.catalyst.util.GenericArrayData(values);
/* 1439 */
/* 1440 */     }
/* 1441 */     ArrayData value165 = null;
/* 1442 */
/* 1443 */     if (!isNull166) {
/* 1444 */
/* 1445 */       indexer.dto.ticket.VendorItemDto[] convertedArray19 = null;
/* 1446 */       int dataLength19 = value166.numElements();
/* 1447 */       convertedArray19 = new indexer.dto.ticket.VendorItemDto[dataLength19];
/* 1448 */
/* 1449 */       int loopIndex19 = 0;
/* 1450 */       while (loopIndex19 &lt; dataLength19) {
/* 1451 */         MapObjects_loopValue378 = (InternalRow) (value166.getStruct(loopIndex19, 21));
/* 1452 */         MapObjects_loopIsNull379 = value166.isNullAt(loopIndex19);
/* 1453 */
/* 1454 */
/* 1455 */         boolean isNull168 = false;
/* 1456 */         indexer.dto.ticket.VendorItemDto value168 = null;
/* 1457 */         if (!false &amp;&amp; MapObjects_loopIsNull379) {
/* 1458 */
/* 1459 */           final indexer.dto.ticket.VendorItemDto value170 = null;
/* 1460 */           isNull168 = true;
/* 1461 */           value168 = value170;
/* 1462 */         } else {
/* 1463 */
/* 1464 */           apply1_0(i);
/* 1465 */           apply1_1(i);
/* 1466 */           apply1_2(i);
/* 1467 */           apply1_3(i);
/* 1468 */           apply1_4(i);
/* 1469 */           apply1_5(i);
/* 1470 */           apply1_6(i);
/* 1471 */           apply1_7(i);
/* 1472 */           apply1_8(i);
/* 1473 */           apply1_9(i);
/* 1474 */           apply1_10(i);
/* 1475 */           apply1_11(i);
/* 1476 */           apply1_12(i);
/* 1477 */
/* 1478 */           final indexer.dto.ticket.VendorItemDto value171 = false ? null : new indexer.dto.ticket.VendorItemDto(argValue92, argValue93, argValue94, argValue95, argValue96, argValue97, argValue98, argValue99, argValue100, argValue101, argValue102, argValue103, argValue104, argValue105, argValue106, argValue107, argValue108, argValue109, argValue110, argValue111, argValue112);
/* 1479 */           isNull168 = false;
/* 1480 */           value168 = value171;
/* 1481 */         }
/* 1482 */         if (isNull168) {
/* 1483 */           convertedArray19[loopIndex19] = null;
/* 1484 */         } else {
/* 1485 */           convertedArray19[loopIndex19] = value168;
/* 1486 */         }
/* 1487 */
/* 1488 */         loopIndex19 += 1;
/* 1489 */       }
/* 1490 */
/* 1491 */       value165 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray19);
/* 1492 */     }
/* 1493 */     boolean isNull164 = true;
/* 1494 */     indexer.dto.ticket.VendorItemDto[] value164 = null;
/* 1495 */     if (!isNull166) {
/* 1496 */
/* 1497 */       isNull164 = false;
/* 1498 */       if (!isNull164) {
/* 1499 */
/* 1500 */         Object funcResult56 = null;
/* 1501 */         funcResult56 = value165.array();
/* 1502 */         if (funcResult56 == null) {
/* 1503 */           isNull164 = true;
/* 1504 */         } else {
/* 1505 */           value164 = (indexer.dto.ticket.VendorItemDto[]) funcResult56;
/* 1506 */         }
/* 1507 */
/* 1508 */       }
/* 1509 */       isNull164 = value164 == null;
/* 1510 */     }
/* 1511 */     argValue37 = value164;
/* 1512 */
/* 1513 */   }


Thanks a lot
Proust



地址: 上海市浦东新区金科路2889弄长泰广场C座7楼 
Address: 7th floor, No#3, Jinke road 2889, Pudong district, Shanghai, China.
Mobile: +86 13621672634
Reply | Threaded
Open this post in threaded view
|

Re: [Spark SQL] NegativeArraySizeException When Parse InternalRow to DTO Field with Type Array[String]

Proust (Feng Guizhou) [Travel Search & Discovery]
Thanks for the information, tried both JavaSerializer and KryoSerializer, same problem encountered

And the stacktrace looks very different from the one mentioned in the stackoverflow link



From: Sandeep Patra <[hidden email]>
Sent: Sunday, February 23, 2020 8:04 PM
To: Proust (Feng Guizhou) [Travel Search & Discovery] <[hidden email]>
Cc: [hidden email] <[hidden email]>
Subject: Re: [Spark SQL] NegativeArraySizeException When Parse InternalRow to DTO Field with Type Array[String]
 
[Warning]: This email originated from an external source. Do not open links or attachments unless you know the content is safe.
[경고]: 본 이메일은 회사 외부에서 유입되었습니다. 내용이 안전한지 확인하기 전까지는 링크나 첨부파일을 열지 마십시오.

This might be due to the serializer being used.

On Sun, Feb 23, 2020 at 1:39 PM Proust (Feng Guizhou) [Travel Search & Discovery] <[hidden email]> wrote:
Hi, Spark Users

I ecounter below NegativeArraySizeException when run Spark SQL. The catalyst generated code for "apply2_19" and "apply1_11" is attached and also the related DTO.
Difficult to understand how the problem could happen, please help if any idea.

I can see maybe https://issues.apache.org/jira/browse/SPARK-15062 is similar but my data type is Array[String] and Spark version is 2.1.2 which looks good both.
By increasing memory to 8G one will instead get a NegativeArraySizeException or a segfault. See here for original discussion: http://apache-spark-developers-list ...


java.lang.NegativeArraySizeException
  at org.apache.spark.unsafe.types.UTF8String.getBytes(UTF8String.java:229)
  at org.apache.spark.unsafe.types.UTF8String.toString(UTF8String.java:1005)
  at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply1_11$(generated.java:2467)
  at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply2_19$(generated.java:1475)
  at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply(generated.java:3881)
  at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
  at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
  at scala.collection.Iterator$GroupedIterator.takeDestructively(Iterator.scala:1076)
  at scala.collection.Iterator$GroupedIterator.go(Iterator.scala:1091)
  at scala.collection.Iterator$GroupedIterator.fill(Iterator.scala:1128)
  at scala.collection.Iterator$GroupedIterator.hasNext(Iterator.scala:1132)
  at scala.collection.Iterator$class.foreach(Iterator.scala:893)
  at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
  at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:101)
  at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:95)
  at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
  at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
  at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
  at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
  at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
  at org.apache.spark.scheduler.Task.run(Task.scala:99)
  at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  at java.lang.Thread.run(Thread.java:748)


case class VendorItemDto(
rate_category_id: java.lang.Long,
product_id: java.lang.Long,
vendor_item_rate_category_id: String,
vendor_item_id: java.lang.Long,
name: String,
item_name: String,
vendor_item_order: java.lang.Integer,
option_codes: Array[String],
option_names: Array[String],
feature_values_ids: Array[String],
feature_values_names: Array[String],
benefit_policy_type: String,
benefit_discount_rate: java.lang.Double,
benefit_discount_amount: java.lang.Long,
available_stock: java.lang.Integer,
sale_price: java.lang.Double,
original_price: java.lang.Double,
supply_price: java.lang.Double,
period_id_set: Array[String],
use_start_set: Array[String],
use_end_set: Array[String]
)


/* 2430 */   private void apply1_11(InternalRow i) {
/* 2431 */
/* 2432 */
/* 2433 */     boolean isNull222 = MapObjects_loopIsNull379;
/* 2434 */     ArrayData value222 = null;
/* 2435 */
/* 2436 */     if (!MapObjects_loopIsNull379) {
/* 2437 */
/* 2438 */       if (MapObjects_loopValue378.isNullAt(19)) {
/* 2439 */         isNull222 = true;
/* 2440 */       } else {
/* 2441 */         value222 = MapObjects_loopValue378.getArray(19);
/* 2442 */       }
/* 2443 */
/* 2444 */     }
/* 2445 */     ArrayData value221 = null;
/* 2446 */
/* 2447 */     if (!isNull222) {
/* 2448 */
/* 2449 */       java.lang.String[] convertedArray17 = null;
/* 2450 */       int dataLength17 = value222.numElements();
/* 2451 */       convertedArray17 = new java.lang.String[dataLength17];
/* 2452 */
/* 2453 */       int loopIndex17 = 0;
/* 2454 */       while (loopIndex17 &lt; dataLength17) {
/* 2455 */         MapObjects_loopValue390 = (UTF8String) (value222.getUTF8String(loopIndex17));
/* 2456 */         MapObjects_loopIsNull391 = value222.isNullAt(loopIndex17);
/* 2457 */
/* 2458 */
/* 2459 */         boolean isNull223 = true;
/* 2460 */         java.lang.String value223 = null;
/* 2461 */         if (!MapObjects_loopIsNull391) {
/* 2462 */
/* 2463 */           isNull223 = false;
/* 2464 */           if (!isNull223) {
/* 2465 */
/* 2466 */             Object funcResult52 = null;
/* 2467 */             funcResult52 = MapObjects_loopValue390.toString();
/* 2468 */             if (funcResult52 == null) {
/* 2469 */               isNull223 = true;
/* 2470 */             } else {
/* 2471 */               value223 = (java.lang.String) funcResult52;
/* 2472 */             }
/* 2473 */
/* 2474 */           }
/* 2475 */           isNull223 = value223 == null;
/* 2476 */         }
/* 2477 */         if (isNull223) {
/* 2478 */           convertedArray17[loopIndex17] = null;
/* 2479 */         } else {
/* 2480 */           convertedArray17[loopIndex17] = value223;
/* 2481 */         }
/* 2482 */
/* 2483 */         loopIndex17 += 1;
/* 2484 */       }
/* 2485 */
/* 2486 */       value221 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray17);
/* 2487 */     }
/* 2488 */     boolean isNull220 = true;
/* 2489 */     java.lang.String[] value220 = null;
/* 2490 */     if (!isNull222) {
/* 2491 */
/* 2492 */       isNull220 = false;
/* 2493 */       if (!isNull220) {
/* 2494 */
/* 2495 */         Object funcResult53 = null;
/* 2496 */         funcResult53 = value221.array();
/* 2497 */         if (funcResult53 == null) {
/* 2498 */           isNull220 = true;
/* 2499 */         } else {
/* 2500 */           value220 = (java.lang.String[]) funcResult53;
/* 2501 */         }
/* 2502 */
/* 2503 */       }
/* 2504 */       isNull220 = value220 == null;
/* 2505 */     }
/* 2506 */     argValue111 = value220;
/* 2507 */
/* 2508 */   }


/* 933 */   private void apply2_19(InternalRow i) {
/* 934 */
/* 935 */
/* 936 */     resultIsNull30 = false;
/* 937 */     if (!resultIsNull30) {
/* 938 */
/* 939 */       boolean isNull161 = i.isNullAt(36);
/* 940 */       long value161 = isNull161 ? -1L : (i.getLong(36));
/* 941 */       resultIsNull30 = isNull161;
/* 942 */       argValue90 = value161;
/* 943 */     }
/* 944 */
/* 945 */
/* 946 */     final java.lang.Long value160 = resultIsNull30 ? null : new java.lang.Long(argValue90);
/* 947 */     argValue35 = value160;
/* 948 */
/* 949 */
/* 950 */     resultIsNull31 = false;
/* 951 */     if (!resultIsNull31) {
/* 952 */
/* 953 */       boolean isNull163 = i.isNullAt(37);
/* 954 */       long value163 = isNull163 ? -1L : (i.getLong(37));
/* 955 */       resultIsNull31 = isNull163;
/* 956 */       argValue91 = value163;
/* 957 */     }
/* 958 */
/* 959 */
/* 960 */     final java.lang.Long value162 = resultIsNull31 ? null : new java.lang.Long(argValue91);
/* 961 */     argValue36 = value162;
/* 962 */
/* 963 */
/* 964 */     boolean isNull167 = i.isNullAt(35);
/* 965 */     ArrayData value167 = isNull167 ? null : (i.getArray(35));
/* 966 */     boolean isNull166 = isNull167;
/* 967 */     ArrayData value166 = null;
/* 968 */     if (!isNull167) {
/* 969 */
/* 970 */       final int n = value167.numElements();
/* 971 */       final Object[] values = new Object[n];
/* 972 */       for (int j = 0; j &lt; n; j ++) {
/* 973 */         if (value167.isNullAt(j)) {
/* 974 */           values[j] = null;
/* 975 */         } else {
/* 976 */           boolean feNull = false;
/* 977 */           InternalRow fePrim =
/* 978 */           value167.getStruct(j, 21);
/* 979 */
/* 980 */           boolean teNull = feNull;
/* 981 */           InternalRow tePrim = null;
/* 982 */           if (!feNull) {
/* 983 */
/* 984 */             final org.apache.spark.sql.catalyst.expressions.GenericInternalRow result = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(21);
/* 985 */             final InternalRow tmpRow = fePrim;
/* 986 */
/* 987 */             boolean ffn = tmpRow.isNullAt(0);
/* 988 */             if (ffn) {
/* 989 */               result.setNullAt(0);
/* 990 */             } else {
/* 991 */               long ffp =
/* 992 */               tmpRow.getLong(0);
/* 993 */
/* 994 */               boolean tfn = ffn;
/* 995 */               long tfp = -1L;
/* 996 */               if (!ffn) {
/* 997 */                 tfp = ffp;
/* 998 */               }
/* 999 */
/* 1000 */               if (tfn) {
/* 1001 */                 result.setNullAt(0);
/* 1002 */               } else {
/* 1003 */                 result.setLong(0, tfp);
/* 1004 */               }
/* 1005 */             }
/* 1006 */
/* 1007 */
/* 1008 */             boolean ffn1 = tmpRow.isNullAt(1);
/* 1009 */             if (ffn1) {
/* 1010 */               result.setNullAt(1);
/* 1011 */             } else {
/* 1012 */               long ffp1 =
/* 1013 */               tmpRow.getLong(1);
/* 1014 */
/* 1015 */               boolean tfn1 = ffn1;
/* 1016 */               long tfp1 = -1L;
/* 1017 */               if (!ffn1) {
/* 1018 */                 tfp1 = ffp1;
/* 1019 */               }
/* 1020 */
/* 1021 */               if (tfn1) {
/* 1022 */                 result.setNullAt(1);
/* 1023 */               } else {
/* 1024 */                 result.setLong(1, tfp1);
/* 1025 */               }
/* 1026 */             }
/* 1027 */
/* 1028 */
/* 1029 */             boolean ffn2 = tmpRow.isNullAt(2);
/* 1030 */             if (ffn2) {
/* 1031 */               result.setNullAt(2);
/* 1032 */             } else {
/* 1033 */               UTF8String ffp2 =
/* 1034 */               tmpRow.getUTF8String(2);
/* 1035 */
/* 1036 */               boolean tfn2 = ffn2;
/* 1037 */               UTF8String tfp2 = null;
/* 1038 */               if (!ffn2) {
/* 1039 */                 tfp2 = ffp2;
/* 1040 */               }
/* 1041 */
/* 1042 */               if (tfn2) {
/* 1043 */                 result.setNullAt(2);
/* 1044 */               } else {
/* 1045 */                 result.update(2, tfp2.clone());
/* 1046 */               }
/* 1047 */             }
/* 1048 */
/* 1049 */
/* 1050 */             boolean ffn3 = tmpRow.isNullAt(3);
/* 1051 */             if (ffn3) {
/* 1052 */               result.setNullAt(3);
/* 1053 */             } else {
/* 1054 */               long ffp3 =
/* 1055 */               tmpRow.getLong(3);
/* 1056 */
/* 1057 */               boolean tfn3 = ffn3;
/* 1058 */               long tfp3 = -1L;
/* 1059 */               if (!ffn3) {
/* 1060 */                 tfp3 = ffp3;
/* 1061 */               }
/* 1062 */
/* 1063 */               if (tfn3) {
/* 1064 */                 result.setNullAt(3);
/* 1065 */               } else {
/* 1066 */                 result.setLong(3, tfp3);
/* 1067 */               }
/* 1068 */             }
/* 1069 */
/* 1070 */
/* 1071 */             boolean ffn4 = tmpRow.isNullAt(4);
/* 1072 */             if (ffn4) {
/* 1073 */               result.setNullAt(4);
/* 1074 */             } else {
/* 1075 */               UTF8String ffp4 =
/* 1076 */               tmpRow.getUTF8String(4);
/* 1077 */
/* 1078 */               boolean tfn4 = ffn4;
/* 1079 */               UTF8String tfp4 = null;
/* 1080 */               if (!ffn4) {
/* 1081 */                 tfp4 = ffp4;
/* 1082 */               }
/* 1083 */
/* 1084 */               if (tfn4) {
/* 1085 */                 result.setNullAt(4);
/* 1086 */               } else {
/* 1087 */                 result.update(4, tfp4.clone());
/* 1088 */               }
/* 1089 */             }
/* 1090 */
/* 1091 */
/* 1092 */             boolean ffn5 = tmpRow.isNullAt(5);
/* 1093 */             if (ffn5) {
/* 1094 */               result.setNullAt(5);
/* 1095 */             } else {
/* 1096 */               UTF8String ffp5 =
/* 1097 */               tmpRow.getUTF8String(5);
/* 1098 */
/* 1099 */               boolean tfn5 = ffn5;
/* 1100 */               UTF8String tfp5 = null;
/* 1101 */               if (!ffn5) {
/* 1102 */                 tfp5 = ffp5;
/* 1103 */               }
/* 1104 */
/* 1105 */               if (tfn5) {
/* 1106 */                 result.setNullAt(5);
/* 1107 */               } else {
/* 1108 */                 result.update(5, tfp5.clone());
/* 1109 */               }
/* 1110 */             }
/* 1111 */
/* 1112 */
/* 1113 */             boolean ffn6 = tmpRow.isNullAt(6);
/* 1114 */             if (ffn6) {
/* 1115 */               result.setNullAt(6);
/* 1116 */             } else {
/* 1117 */               int ffp6 =
/* 1118 */               tmpRow.getInt(6);
/* 1119 */
/* 1120 */               boolean tfn6 = ffn6;
/* 1121 */               int tfp6 = -1;
/* 1122 */               if (!ffn6) {
/* 1123 */                 tfp6 = ffp6;
/* 1124 */               }
/* 1125 */
/* 1126 */               if (tfn6) {
/* 1127 */                 result.setNullAt(6);
/* 1128 */               } else {
/* 1129 */                 result.setInt(6, tfp6);
/* 1130 */               }
/* 1131 */             }
/* 1132 */
/* 1133 */
/* 1134 */             boolean ffn7 = tmpRow.isNullAt(7);
/* 1135 */             if (ffn7) {
/* 1136 */               result.setNullAt(7);
/* 1137 */             } else {
/* 1138 */               ArrayData ffp7 =
/* 1139 */               tmpRow.getArray(7);
/* 1140 */
/* 1141 */               boolean tfn7 = ffn7;
/* 1142 */               ArrayData tfp7 = null;
/* 1143 */               if (!ffn7) {
/* 1144 */                 tfp7 = ffp7;
/* 1145 */               }
/* 1146 */
/* 1147 */               if (tfn7) {
/* 1148 */                 result.setNullAt(7);
/* 1149 */               } else {
/* 1150 */                 result.update(7, tfp7);
/* 1151 */               }
/* 1152 */             }
/* 1153 */
/* 1154 */
/* 1155 */             boolean ffn8 = tmpRow.isNullAt(8);
/* 1156 */             if (ffn8) {
/* 1157 */               result.setNullAt(8);
/* 1158 */             } else {
/* 1159 */               ArrayData ffp8 =
/* 1160 */               tmpRow.getArray(8);
/* 1161 */
/* 1162 */               boolean tfn8 = ffn8;
/* 1163 */               ArrayData tfp8 = null;
/* 1164 */               if (!ffn8) {
/* 1165 */                 tfp8 = ffp8;
/* 1166 */               }
/* 1167 */
/* 1168 */               if (tfn8) {
/* 1169 */                 result.setNullAt(8);
/* 1170 */               } else {
/* 1171 */                 result.update(8, tfp8);
/* 1172 */               }
/* 1173 */             }
/* 1174 */
/* 1175 */
/* 1176 */             boolean ffn9 = tmpRow.isNullAt(9);
/* 1177 */             if (ffn9) {
/* 1178 */               result.setNullAt(9);
/* 1179 */             } else {
/* 1180 */               ArrayData ffp9 =
/* 1181 */               tmpRow.getArray(9);
/* 1182 */
/* 1183 */               boolean tfn9 = ffn9;
/* 1184 */               ArrayData tfp9 = null;
/* 1185 */               if (!ffn9) {
/* 1186 */                 tfp9 = ffp9;
/* 1187 */               }
/* 1188 */
/* 1189 */               if (tfn9) {
/* 1190 */                 result.setNullAt(9);
/* 1191 */               } else {
/* 1192 */                 result.update(9, tfp9);
/* 1193 */               }
/* 1194 */             }
/* 1195 */
/* 1196 */
/* 1197 */             boolean ffn10 = tmpRow.isNullAt(10);
/* 1198 */             if (ffn10) {
/* 1199 */               result.setNullAt(10);
/* 1200 */             } else {
/* 1201 */               ArrayData ffp10 =
/* 1202 */               tmpRow.getArray(10);
/* 1203 */
/* 1204 */               boolean tfn10 = ffn10;
/* 1205 */               ArrayData tfp10 = null;
/* 1206 */               if (!ffn10) {
/* 1207 */                 tfp10 = ffp10;
/* 1208 */               }
/* 1209 */
/* 1210 */               if (tfn10) {
/* 1211 */                 result.setNullAt(10);
/* 1212 */               } else {
/* 1213 */                 result.update(10, tfp10);
/* 1214 */               }
/* 1215 */             }
/* 1216 */
/* 1217 */
/* 1218 */             boolean ffn11 = tmpRow.isNullAt(11);
/* 1219 */             if (ffn11) {
/* 1220 */               result.setNullAt(11);
/* 1221 */             } else {
/* 1222 */               UTF8String ffp11 =
/* 1223 */               tmpRow.getUTF8String(11);
/* 1224 */
/* 1225 */               boolean tfn11 = ffn11;
/* 1226 */               UTF8String tfp11 = null;
/* 1227 */               if (!ffn11) {
/* 1228 */                 tfp11 = ffp11;
/* 1229 */               }
/* 1230 */
/* 1231 */               if (tfn11) {
/* 1232 */                 result.setNullAt(11);
/* 1233 */               } else {
/* 1234 */                 result.update(11, tfp11.clone());
/* 1235 */               }
/* 1236 */             }
/* 1237 */
/* 1238 */
/* 1239 */             boolean ffn12 = tmpRow.isNullAt(12);
/* 1240 */             if (ffn12) {
/* 1241 */               result.setNullAt(12);
/* 1242 */             } else {
/* 1243 */               double ffp12 =
/* 1244 */               tmpRow.getDouble(12);
/* 1245 */
/* 1246 */               boolean tfn12 = ffn12;
/* 1247 */               double tfp12 = -1.0;
/* 1248 */               if (!ffn12) {
/* 1249 */                 tfp12 = ffp12;
/* 1250 */               }
/* 1251 */
/* 1252 */               if (tfn12) {
/* 1253 */                 result.setNullAt(12);
/* 1254 */               } else {
/* 1255 */                 result.setDouble(12, tfp12);
/* 1256 */               }
/* 1257 */             }
/* 1258 */
/* 1259 */
/* 1260 */             boolean ffn13 = tmpRow.isNullAt(13);
/* 1261 */             if (ffn13) {
/* 1262 */               result.setNullAt(13);
/* 1263 */             } else {
/* 1264 */               long ffp13 =
/* 1265 */               tmpRow.getLong(13);
/* 1266 */
/* 1267 */               boolean tfn13 = ffn13;
/* 1268 */               long tfp13 = -1L;
/* 1269 */               if (!ffn13) {
/* 1270 */                 tfp13 = ffp13;
/* 1271 */               }
/* 1272 */
/* 1273 */               if (tfn13) {
/* 1274 */                 result.setNullAt(13);
/* 1275 */               } else {
/* 1276 */                 result.setLong(13, tfp13);
/* 1277 */               }
/* 1278 */             }
/* 1279 */
/* 1280 */
/* 1281 */             boolean ffn14 = tmpRow.isNullAt(14);
/* 1282 */             if (ffn14) {
/* 1283 */               result.setNullAt(14);
/* 1284 */             } else {
/* 1285 */               int ffp14 =
/* 1286 */               tmpRow.getInt(14);
/* 1287 */
/* 1288 */               boolean tfn14 = ffn14;
/* 1289 */               int tfp14 = -1;
/* 1290 */               if (!ffn14) {
/* 1291 */                 tfp14 = ffp14;
/* 1292 */               }
/* 1293 */
/* 1294 */               if (tfn14) {
/* 1295 */                 result.setNullAt(14);
/* 1296 */               } else {
/* 1297 */                 result.setInt(14, tfp14);
/* 1298 */               }
/* 1299 */             }
/* 1300 */
/* 1301 */
/* 1302 */             boolean ffn15 = tmpRow.isNullAt(15);
/* 1303 */             if (ffn15) {
/* 1304 */               result.setNullAt(15);
/* 1305 */             } else {
/* 1306 */               double ffp15 =
/* 1307 */               tmpRow.getDouble(15);
/* 1308 */
/* 1309 */               boolean tfn15 = ffn15;
/* 1310 */               double tfp15 = -1.0;
/* 1311 */               if (!ffn15) {
/* 1312 */                 tfp15 = ffp15;
/* 1313 */               }
/* 1314 */
/* 1315 */               if (tfn15) {
/* 1316 */                 result.setNullAt(15);
/* 1317 */               } else {
/* 1318 */                 result.setDouble(15, tfp15);
/* 1319 */               }
/* 1320 */             }
/* 1321 */
/* 1322 */
/* 1323 */             boolean ffn16 = tmpRow.isNullAt(16);
/* 1324 */             if (ffn16) {
/* 1325 */               result.setNullAt(16);
/* 1326 */             } else {
/* 1327 */               double ffp16 =
/* 1328 */               tmpRow.getDouble(16);
/* 1329 */
/* 1330 */               boolean tfn16 = ffn16;
/* 1331 */               double tfp16 = -1.0;
/* 1332 */               if (!ffn16) {
/* 1333 */                 tfp16 = ffp16;
/* 1334 */               }
/* 1335 */
/* 1336 */               if (tfn16) {
/* 1337 */                 result.setNullAt(16);
/* 1338 */               } else {
/* 1339 */                 result.setDouble(16, tfp16);
/* 1340 */               }
/* 1341 */             }
/* 1342 */
/* 1343 */
/* 1344 */             boolean ffn17 = tmpRow.isNullAt(17);
/* 1345 */             if (ffn17) {
/* 1346 */               result.setNullAt(17);
/* 1347 */             } else {
/* 1348 */               double ffp17 =
/* 1349 */               tmpRow.getDouble(17);
/* 1350 */
/* 1351 */               boolean tfn17 = ffn17;
/* 1352 */               double tfp17 = -1.0;
/* 1353 */               if (!ffn17) {
/* 1354 */                 tfp17 = ffp17;
/* 1355 */               }
/* 1356 */
/* 1357 */               if (tfn17) {
/* 1358 */                 result.setNullAt(17);
/* 1359 */               } else {
/* 1360 */                 result.setDouble(17, tfp17);
/* 1361 */               }
/* 1362 */             }
/* 1363 */
/* 1364 */
/* 1365 */             boolean ffn18 = tmpRow.isNullAt(18);
/* 1366 */             if (ffn18) {
/* 1367 */               result.setNullAt(18);
/* 1368 */             } else {
/* 1369 */               ArrayData ffp18 =
/* 1370 */               tmpRow.getArray(18);
/* 1371 */
/* 1372 */               boolean tfn18 = ffn18;
/* 1373 */               ArrayData tfp18 = null;
/* 1374 */               if (!ffn18) {
/* 1375 */                 tfp18 = ffp18;
/* 1376 */               }
/* 1377 */
/* 1378 */               if (tfn18) {
/* 1379 */                 result.setNullAt(18);
/* 1380 */               } else {
/* 1381 */                 result.update(18, tfp18);
/* 1382 */               }
/* 1383 */             }
/* 1384 */
/* 1385 */
/* 1386 */             boolean ffn19 = tmpRow.isNullAt(19);
/* 1387 */             if (ffn19) {
/* 1388 */               result.setNullAt(19);
/* 1389 */             } else {
/* 1390 */               ArrayData ffp19 =
/* 1391 */               tmpRow.getArray(19);
/* 1392 */
/* 1393 */               boolean tfn19 = ffn19;
/* 1394 */               ArrayData tfp19 = null;
/* 1395 */               if (!ffn19) {
/* 1396 */                 tfp19 = ffp19;
/* 1397 */               }
/* 1398 */
/* 1399 */               if (tfn19) {
/* 1400 */                 result.setNullAt(19);
/* 1401 */               } else {
/* 1402 */                 result.update(19, tfp19);
/* 1403 */               }
/* 1404 */             }
/* 1405 */
/* 1406 */
/* 1407 */             boolean ffn20 = tmpRow.isNullAt(20);
/* 1408 */             if (ffn20) {
/* 1409 */               result.setNullAt(20);
/* 1410 */             } else {
/* 1411 */               ArrayData ffp20 =
/* 1412 */               tmpRow.getArray(20);
/* 1413 */
/* 1414 */               boolean tfn20 = ffn20;
/* 1415 */               ArrayData tfp20 = null;
/* 1416 */               if (!ffn20) {
/* 1417 */                 tfp20 = ffp20;
/* 1418 */               }
/* 1419 */
/* 1420 */               if (tfn20) {
/* 1421 */                 result.setNullAt(20);
/* 1422 */               } else {
/* 1423 */                 result.update(20, tfp20);
/* 1424 */               }
/* 1425 */             }
/* 1426 */
/* 1427 */             tePrim = result.copy();
/* 1428 */
/* 1429 */           }
/* 1430 */
/* 1431 */           if (teNull) {
/* 1432 */             values[j] = null;
/* 1433 */           } else {
/* 1434 */             values[j] = tePrim;
/* 1435 */           }
/* 1436 */         }
/* 1437 */       }
/* 1438 */       value166 = new org.apache.spark.sql.catalyst.util.GenericArrayData(values);
/* 1439 */
/* 1440 */     }
/* 1441 */     ArrayData value165 = null;
/* 1442 */
/* 1443 */     if (!isNull166) {
/* 1444 */
/* 1445 */       indexer.dto.ticket.VendorItemDto[] convertedArray19 = null;
/* 1446 */       int dataLength19 = value166.numElements();
/* 1447 */       convertedArray19 = new indexer.dto.ticket.VendorItemDto[dataLength19];
/* 1448 */
/* 1449 */       int loopIndex19 = 0;
/* 1450 */       while (loopIndex19 &lt; dataLength19) {
/* 1451 */         MapObjects_loopValue378 = (InternalRow) (value166.getStruct(loopIndex19, 21));
/* 1452 */         MapObjects_loopIsNull379 = value166.isNullAt(loopIndex19);
/* 1453 */
/* 1454 */
/* 1455 */         boolean isNull168 = false;
/* 1456 */         indexer.dto.ticket.VendorItemDto value168 = null;
/* 1457 */         if (!false &amp;&amp; MapObjects_loopIsNull379) {
/* 1458 */
/* 1459 */           final indexer.dto.ticket.VendorItemDto value170 = null;
/* 1460 */           isNull168 = true;
/* 1461 */           value168 = value170;
/* 1462 */         } else {
/* 1463 */
/* 1464 */           apply1_0(i);
/* 1465 */           apply1_1(i);
/* 1466 */           apply1_2(i);
/* 1467 */           apply1_3(i);
/* 1468 */           apply1_4(i);
/* 1469 */           apply1_5(i);
/* 1470 */           apply1_6(i);
/* 1471 */           apply1_7(i);
/* 1472 */           apply1_8(i);
/* 1473 */           apply1_9(i);
/* 1474 */           apply1_10(i);
/* 1475 */           apply1_11(i);
/* 1476 */           apply1_12(i);
/* 1477 */
/* 1478 */           final indexer.dto.ticket.VendorItemDto value171 = false ? null : new indexer.dto.ticket.VendorItemDto(argValue92, argValue93, argValue94, argValue95, argValue96, argValue97, argValue98, argValue99, argValue100, argValue101, argValue102, argValue103, argValue104, argValue105, argValue106, argValue107, argValue108, argValue109, argValue110, argValue111, argValue112);
/* 1479 */           isNull168 = false;
/* 1480 */           value168 = value171;
/* 1481 */         }
/* 1482 */         if (isNull168) {
/* 1483 */           convertedArray19[loopIndex19] = null;
/* 1484 */         } else {
/* 1485 */           convertedArray19[loopIndex19] = value168;
/* 1486 */         }
/* 1487 */
/* 1488 */         loopIndex19 += 1;
/* 1489 */       }
/* 1490 */
/* 1491 */       value165 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray19);
/* 1492 */     }
/* 1493 */     boolean isNull164 = true;
/* 1494 */     indexer.dto.ticket.VendorItemDto[] value164 = null;
/* 1495 */     if (!isNull166) {
/* 1496 */
/* 1497 */       isNull164 = false;
/* 1498 */       if (!isNull164) {
/* 1499 */
/* 1500 */         Object funcResult56 = null;
/* 1501 */         funcResult56 = value165.array();
/* 1502 */         if (funcResult56 == null) {
/* 1503 */           isNull164 = true;
/* 1504 */         } else {
/* 1505 */           value164 = (indexer.dto.ticket.VendorItemDto[]) funcResult56;
/* 1506 */         }
/* 1507 */
/* 1508 */       }
/* 1509 */       isNull164 = value164 == null;
/* 1510 */     }
/* 1511 */     argValue37 = value164;
/* 1512 */
/* 1513 */   }


Thanks a lot
Proust



地址: 上海市浦东新区金科路2889弄长泰广场C座7楼 
Address: 7th floor, No#3, Jinke road 2889, Pudong district, Shanghai, China.
Mobile: +86 13621672634
Reply | Threaded
Open this post in threaded view
|

Re: [Spark SQL] NegativeArraySizeException When Parse InternalRow to DTO Field with Type Array[String]

Proust (Feng Guizhou) [Travel Search & Discovery]
Also I tried to disable the Kryo reference tracking, then the problem will simply change to Java stack overflow exception.

From: Proust (Feng Guizhou) [Travel Search & Discovery] <[hidden email]>
Sent: Tuesday, February 25, 2020 11:28 PM
To: Sandeep Patra <[hidden email]>
Cc: [hidden email] <[hidden email]>
Subject: Re: [Spark SQL] NegativeArraySizeException When Parse InternalRow to DTO Field with Type Array[String]
 
Thanks for the information, tried both JavaSerializer and KryoSerializer, same problem encountered

And the stacktrace looks very different from the one mentioned in the stackoverflow link



From: Sandeep Patra <[hidden email]>
Sent: Sunday, February 23, 2020 8:04 PM
To: Proust (Feng Guizhou) [Travel Search & Discovery] <[hidden email]>
Cc: [hidden email] <[hidden email]>
Subject: Re: [Spark SQL] NegativeArraySizeException When Parse InternalRow to DTO Field with Type Array[String]
 
[Warning]: This email originated from an external source. Do not open links or attachments unless you know the content is safe.
[경고]: 본 이메일은 회사 외부에서 유입되었습니다. 내용이 안전한지 확인하기 전까지는 링크나 첨부파일을 열지 마십시오.

This might be due to the serializer being used.

On Sun, Feb 23, 2020 at 1:39 PM Proust (Feng Guizhou) [Travel Search & Discovery] <[hidden email]> wrote:
Hi, Spark Users

I ecounter below NegativeArraySizeException when run Spark SQL. The catalyst generated code for "apply2_19" and "apply1_11" is attached and also the related DTO.
Difficult to understand how the problem could happen, please help if any idea.

I can see maybe https://issues.apache.org/jira/browse/SPARK-15062 is similar but my data type is Array[String] and Spark version is 2.1.2 which looks good both.
By increasing memory to 8G one will instead get a NegativeArraySizeException or a segfault. See here for original discussion: http://apache-spark-developers-list ...


java.lang.NegativeArraySizeException
  at org.apache.spark.unsafe.types.UTF8String.getBytes(UTF8String.java:229)
  at org.apache.spark.unsafe.types.UTF8String.toString(UTF8String.java:1005)
  at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply1_11$(generated.java:2467)
  at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply2_19$(generated.java:1475)
  at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply(generated.java:3881)
  at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
  at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
  at scala.collection.Iterator$GroupedIterator.takeDestructively(Iterator.scala:1076)
  at scala.collection.Iterator$GroupedIterator.go(Iterator.scala:1091)
  at scala.collection.Iterator$GroupedIterator.fill(Iterator.scala:1128)
  at scala.collection.Iterator$GroupedIterator.hasNext(Iterator.scala:1132)
  at scala.collection.Iterator$class.foreach(Iterator.scala:893)
  at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
  at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:101)
  at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:95)
  at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
  at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
  at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
  at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
  at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
  at org.apache.spark.scheduler.Task.run(Task.scala:99)
  at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  at java.lang.Thread.run(Thread.java:748)


case class VendorItemDto(
rate_category_id: java.lang.Long,
product_id: java.lang.Long,
vendor_item_rate_category_id: String,
vendor_item_id: java.lang.Long,
name: String,
item_name: String,
vendor_item_order: java.lang.Integer,
option_codes: Array[String],
option_names: Array[String],
feature_values_ids: Array[String],
feature_values_names: Array[String],
benefit_policy_type: String,
benefit_discount_rate: java.lang.Double,
benefit_discount_amount: java.lang.Long,
available_stock: java.lang.Integer,
sale_price: java.lang.Double,
original_price: java.lang.Double,
supply_price: java.lang.Double,
period_id_set: Array[String],
use_start_set: Array[String],
use_end_set: Array[String]
)


/* 2430 */   private void apply1_11(InternalRow i) {
/* 2431 */
/* 2432 */
/* 2433 */     boolean isNull222 = MapObjects_loopIsNull379;
/* 2434 */     ArrayData value222 = null;
/* 2435 */
/* 2436 */     if (!MapObjects_loopIsNull379) {
/* 2437 */
/* 2438 */       if (MapObjects_loopValue378.isNullAt(19)) {
/* 2439 */         isNull222 = true;
/* 2440 */       } else {
/* 2441 */         value222 = MapObjects_loopValue378.getArray(19);
/* 2442 */       }
/* 2443 */
/* 2444 */     }
/* 2445 */     ArrayData value221 = null;
/* 2446 */
/* 2447 */     if (!isNull222) {
/* 2448 */
/* 2449 */       java.lang.String[] convertedArray17 = null;
/* 2450 */       int dataLength17 = value222.numElements();
/* 2451 */       convertedArray17 = new java.lang.String[dataLength17];
/* 2452 */
/* 2453 */       int loopIndex17 = 0;
/* 2454 */       while (loopIndex17 &lt; dataLength17) {
/* 2455 */         MapObjects_loopValue390 = (UTF8String) (value222.getUTF8String(loopIndex17));
/* 2456 */         MapObjects_loopIsNull391 = value222.isNullAt(loopIndex17);
/* 2457 */
/* 2458 */
/* 2459 */         boolean isNull223 = true;
/* 2460 */         java.lang.String value223 = null;
/* 2461 */         if (!MapObjects_loopIsNull391) {
/* 2462 */
/* 2463 */           isNull223 = false;
/* 2464 */           if (!isNull223) {
/* 2465 */
/* 2466 */             Object funcResult52 = null;
/* 2467 */             funcResult52 = MapObjects_loopValue390.toString();
/* 2468 */             if (funcResult52 == null) {
/* 2469 */               isNull223 = true;
/* 2470 */             } else {
/* 2471 */               value223 = (java.lang.String) funcResult52;
/* 2472 */             }
/* 2473 */
/* 2474 */           }
/* 2475 */           isNull223 = value223 == null;
/* 2476 */         }
/* 2477 */         if (isNull223) {
/* 2478 */           convertedArray17[loopIndex17] = null;
/* 2479 */         } else {
/* 2480 */           convertedArray17[loopIndex17] = value223;
/* 2481 */         }
/* 2482 */
/* 2483 */         loopIndex17 += 1;
/* 2484 */       }
/* 2485 */
/* 2486 */       value221 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray17);
/* 2487 */     }
/* 2488 */     boolean isNull220 = true;
/* 2489 */     java.lang.String[] value220 = null;
/* 2490 */     if (!isNull222) {
/* 2491 */
/* 2492 */       isNull220 = false;
/* 2493 */       if (!isNull220) {
/* 2494 */
/* 2495 */         Object funcResult53 = null;
/* 2496 */         funcResult53 = value221.array();
/* 2497 */         if (funcResult53 == null) {
/* 2498 */           isNull220 = true;
/* 2499 */         } else {
/* 2500 */           value220 = (java.lang.String[]) funcResult53;
/* 2501 */         }
/* 2502 */
/* 2503 */       }
/* 2504 */       isNull220 = value220 == null;
/* 2505 */     }
/* 2506 */     argValue111 = value220;
/* 2507 */
/* 2508 */   }


/* 933 */   private void apply2_19(InternalRow i) {
/* 934 */
/* 935 */
/* 936 */     resultIsNull30 = false;
/* 937 */     if (!resultIsNull30) {
/* 938 */
/* 939 */       boolean isNull161 = i.isNullAt(36);
/* 940 */       long value161 = isNull161 ? -1L : (i.getLong(36));
/* 941 */       resultIsNull30 = isNull161;
/* 942 */       argValue90 = value161;
/* 943 */     }
/* 944 */
/* 945 */
/* 946 */     final java.lang.Long value160 = resultIsNull30 ? null : new java.lang.Long(argValue90);
/* 947 */     argValue35 = value160;
/* 948 */
/* 949 */
/* 950 */     resultIsNull31 = false;
/* 951 */     if (!resultIsNull31) {
/* 952 */
/* 953 */       boolean isNull163 = i.isNullAt(37);
/* 954 */       long value163 = isNull163 ? -1L : (i.getLong(37));
/* 955 */       resultIsNull31 = isNull163;
/* 956 */       argValue91 = value163;
/* 957 */     }
/* 958 */
/* 959 */
/* 960 */     final java.lang.Long value162 = resultIsNull31 ? null : new java.lang.Long(argValue91);
/* 961 */     argValue36 = value162;
/* 962 */
/* 963 */
/* 964 */     boolean isNull167 = i.isNullAt(35);
/* 965 */     ArrayData value167 = isNull167 ? null : (i.getArray(35));
/* 966 */     boolean isNull166 = isNull167;
/* 967 */     ArrayData value166 = null;
/* 968 */     if (!isNull167) {
/* 969 */
/* 970 */       final int n = value167.numElements();
/* 971 */       final Object[] values = new Object[n];
/* 972 */       for (int j = 0; j &lt; n; j ++) {
/* 973 */         if (value167.isNullAt(j)) {
/* 974 */           values[j] = null;
/* 975 */         } else {
/* 976 */           boolean feNull = false;
/* 977 */           InternalRow fePrim =
/* 978 */           value167.getStruct(j, 21);
/* 979 */
/* 980 */           boolean teNull = feNull;
/* 981 */           InternalRow tePrim = null;
/* 982 */           if (!feNull) {
/* 983 */
/* 984 */             final org.apache.spark.sql.catalyst.expressions.GenericInternalRow result = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(21);
/* 985 */             final InternalRow tmpRow = fePrim;
/* 986 */
/* 987 */             boolean ffn = tmpRow.isNullAt(0);
/* 988 */             if (ffn) {
/* 989 */               result.setNullAt(0);
/* 990 */             } else {
/* 991 */               long ffp =
/* 992 */               tmpRow.getLong(0);
/* 993 */
/* 994 */               boolean tfn = ffn;
/* 995 */               long tfp = -1L;
/* 996 */               if (!ffn) {
/* 997 */                 tfp = ffp;
/* 998 */               }
/* 999 */
/* 1000 */               if (tfn) {
/* 1001 */                 result.setNullAt(0);
/* 1002 */               } else {
/* 1003 */                 result.setLong(0, tfp);
/* 1004 */               }
/* 1005 */             }
/* 1006 */
/* 1007 */
/* 1008 */             boolean ffn1 = tmpRow.isNullAt(1);
/* 1009 */             if (ffn1) {
/* 1010 */               result.setNullAt(1);
/* 1011 */             } else {
/* 1012 */               long ffp1 =
/* 1013 */               tmpRow.getLong(1);
/* 1014 */
/* 1015 */               boolean tfn1 = ffn1;
/* 1016 */               long tfp1 = -1L;
/* 1017 */               if (!ffn1) {
/* 1018 */                 tfp1 = ffp1;
/* 1019 */               }
/* 1020 */
/* 1021 */               if (tfn1) {
/* 1022 */                 result.setNullAt(1);
/* 1023 */               } else {
/* 1024 */                 result.setLong(1, tfp1);
/* 1025 */               }
/* 1026 */             }
/* 1027 */
/* 1028 */
/* 1029 */             boolean ffn2 = tmpRow.isNullAt(2);
/* 1030 */             if (ffn2) {
/* 1031 */               result.setNullAt(2);
/* 1032 */             } else {
/* 1033 */               UTF8String ffp2 =
/* 1034 */               tmpRow.getUTF8String(2);
/* 1035 */
/* 1036 */               boolean tfn2 = ffn2;
/* 1037 */               UTF8String tfp2 = null;
/* 1038 */               if (!ffn2) {
/* 1039 */                 tfp2 = ffp2;
/* 1040 */               }
/* 1041 */
/* 1042 */               if (tfn2) {
/* 1043 */                 result.setNullAt(2);
/* 1044 */               } else {
/* 1045 */                 result.update(2, tfp2.clone());
/* 1046 */               }
/* 1047 */             }
/* 1048 */
/* 1049 */
/* 1050 */             boolean ffn3 = tmpRow.isNullAt(3);
/* 1051 */             if (ffn3) {
/* 1052 */               result.setNullAt(3);
/* 1053 */             } else {
/* 1054 */               long ffp3 =
/* 1055 */               tmpRow.getLong(3);
/* 1056 */
/* 1057 */               boolean tfn3 = ffn3;
/* 1058 */               long tfp3 = -1L;
/* 1059 */               if (!ffn3) {
/* 1060 */                 tfp3 = ffp3;
/* 1061 */               }
/* 1062 */
/* 1063 */               if (tfn3) {
/* 1064 */                 result.setNullAt(3);
/* 1065 */               } else {
/* 1066 */                 result.setLong(3, tfp3);
/* 1067 */               }
/* 1068 */             }
/* 1069 */
/* 1070 */
/* 1071 */             boolean ffn4 = tmpRow.isNullAt(4);
/* 1072 */             if (ffn4) {
/* 1073 */               result.setNullAt(4);
/* 1074 */             } else {
/* 1075 */               UTF8String ffp4 =
/* 1076 */               tmpRow.getUTF8String(4);
/* 1077 */
/* 1078 */               boolean tfn4 = ffn4;
/* 1079 */               UTF8String tfp4 = null;
/* 1080 */               if (!ffn4) {
/* 1081 */                 tfp4 = ffp4;
/* 1082 */               }
/* 1083 */
/* 1084 */               if (tfn4) {
/* 1085 */                 result.setNullAt(4);
/* 1086 */               } else {
/* 1087 */                 result.update(4, tfp4.clone());
/* 1088 */               }
/* 1089 */             }
/* 1090 */
/* 1091 */
/* 1092 */             boolean ffn5 = tmpRow.isNullAt(5);
/* 1093 */             if (ffn5) {
/* 1094 */               result.setNullAt(5);
/* 1095 */             } else {
/* 1096 */               UTF8String ffp5 =
/* 1097 */               tmpRow.getUTF8String(5);
/* 1098 */
/* 1099 */               boolean tfn5 = ffn5;
/* 1100 */               UTF8String tfp5 = null;
/* 1101 */               if (!ffn5) {
/* 1102 */                 tfp5 = ffp5;
/* 1103 */               }
/* 1104 */
/* 1105 */               if (tfn5) {
/* 1106 */                 result.setNullAt(5);
/* 1107 */               } else {
/* 1108 */                 result.update(5, tfp5.clone());
/* 1109 */               }
/* 1110 */             }
/* 1111 */
/* 1112 */
/* 1113 */             boolean ffn6 = tmpRow.isNullAt(6);
/* 1114 */             if (ffn6) {
/* 1115 */               result.setNullAt(6);
/* 1116 */             } else {
/* 1117 */               int ffp6 =
/* 1118 */               tmpRow.getInt(6);
/* 1119 */
/* 1120 */               boolean tfn6 = ffn6;
/* 1121 */               int tfp6 = -1;
/* 1122 */               if (!ffn6) {
/* 1123 */                 tfp6 = ffp6;
/* 1124 */               }
/* 1125 */
/* 1126 */               if (tfn6) {
/* 1127 */                 result.setNullAt(6);
/* 1128 */               } else {
/* 1129 */                 result.setInt(6, tfp6);
/* 1130 */               }
/* 1131 */             }
/* 1132 */
/* 1133 */
/* 1134 */             boolean ffn7 = tmpRow.isNullAt(7);
/* 1135 */             if (ffn7) {
/* 1136 */               result.setNullAt(7);
/* 1137 */             } else {
/* 1138 */               ArrayData ffp7 =
/* 1139 */               tmpRow.getArray(7);
/* 1140 */
/* 1141 */               boolean tfn7 = ffn7;
/* 1142 */               ArrayData tfp7 = null;
/* 1143 */               if (!ffn7) {
/* 1144 */                 tfp7 = ffp7;
/* 1145 */               }
/* 1146 */
/* 1147 */               if (tfn7) {
/* 1148 */                 result.setNullAt(7);
/* 1149 */               } else {
/* 1150 */                 result.update(7, tfp7);
/* 1151 */               }
/* 1152 */             }
/* 1153 */
/* 1154 */
/* 1155 */             boolean ffn8 = tmpRow.isNullAt(8);
/* 1156 */             if (ffn8) {
/* 1157 */               result.setNullAt(8);
/* 1158 */             } else {
/* 1159 */               ArrayData ffp8 =
/* 1160 */               tmpRow.getArray(8);
/* 1161 */
/* 1162 */               boolean tfn8 = ffn8;
/* 1163 */               ArrayData tfp8 = null;
/* 1164 */               if (!ffn8) {
/* 1165 */                 tfp8 = ffp8;
/* 1166 */               }
/* 1167 */
/* 1168 */               if (tfn8) {
/* 1169 */                 result.setNullAt(8);
/* 1170 */               } else {
/* 1171 */                 result.update(8, tfp8);
/* 1172 */               }
/* 1173 */             }
/* 1174 */
/* 1175 */
/* 1176 */             boolean ffn9 = tmpRow.isNullAt(9);
/* 1177 */             if (ffn9) {
/* 1178 */               result.setNullAt(9);
/* 1179 */             } else {
/* 1180 */               ArrayData ffp9 =
/* 1181 */               tmpRow.getArray(9);
/* 1182 */
/* 1183 */               boolean tfn9 = ffn9;
/* 1184 */               ArrayData tfp9 = null;
/* 1185 */               if (!ffn9) {
/* 1186 */                 tfp9 = ffp9;
/* 1187 */               }
/* 1188 */
/* 1189 */               if (tfn9) {
/* 1190 */                 result.setNullAt(9);
/* 1191 */               } else {
/* 1192 */                 result.update(9, tfp9);
/* 1193 */               }
/* 1194 */             }
/* 1195 */
/* 1196 */
/* 1197 */             boolean ffn10 = tmpRow.isNullAt(10);
/* 1198 */             if (ffn10) {
/* 1199 */               result.setNullAt(10);
/* 1200 */             } else {
/* 1201 */               ArrayData ffp10 =
/* 1202 */               tmpRow.getArray(10);
/* 1203 */
/* 1204 */               boolean tfn10 = ffn10;
/* 1205 */               ArrayData tfp10 = null;
/* 1206 */               if (!ffn10) {
/* 1207 */                 tfp10 = ffp10;
/* 1208 */               }
/* 1209 */
/* 1210 */               if (tfn10) {
/* 1211 */                 result.setNullAt(10);
/* 1212 */               } else {
/* 1213 */                 result.update(10, tfp10);
/* 1214 */               }
/* 1215 */             }
/* 1216 */
/* 1217 */
/* 1218 */             boolean ffn11 = tmpRow.isNullAt(11);
/* 1219 */             if (ffn11) {
/* 1220 */               result.setNullAt(11);
/* 1221 */             } else {
/* 1222 */               UTF8String ffp11 =
/* 1223 */               tmpRow.getUTF8String(11);
/* 1224 */
/* 1225 */               boolean tfn11 = ffn11;
/* 1226 */               UTF8String tfp11 = null;
/* 1227 */               if (!ffn11) {
/* 1228 */                 tfp11 = ffp11;
/* 1229 */               }
/* 1230 */
/* 1231 */               if (tfn11) {
/* 1232 */                 result.setNullAt(11);
/* 1233 */               } else {
/* 1234 */                 result.update(11, tfp11.clone());
/* 1235 */               }
/* 1236 */             }
/* 1237 */
/* 1238 */
/* 1239 */             boolean ffn12 = tmpRow.isNullAt(12);
/* 1240 */             if (ffn12) {
/* 1241 */               result.setNullAt(12);
/* 1242 */             } else {
/* 1243 */               double ffp12 =
/* 1244 */               tmpRow.getDouble(12);
/* 1245 */
/* 1246 */               boolean tfn12 = ffn12;
/* 1247 */               double tfp12 = -1.0;
/* 1248 */               if (!ffn12) {
/* 1249 */                 tfp12 = ffp12;
/* 1250 */               }
/* 1251 */
/* 1252 */               if (tfn12) {
/* 1253 */                 result.setNullAt(12);
/* 1254 */               } else {
/* 1255 */                 result.setDouble(12, tfp12);
/* 1256 */               }
/* 1257 */             }
/* 1258 */
/* 1259 */
/* 1260 */             boolean ffn13 = tmpRow.isNullAt(13);
/* 1261 */             if (ffn13) {
/* 1262 */               result.setNullAt(13);
/* 1263 */             } else {
/* 1264 */               long ffp13 =
/* 1265 */               tmpRow.getLong(13);
/* 1266 */
/* 1267 */               boolean tfn13 = ffn13;
/* 1268 */               long tfp13 = -1L;
/* 1269 */               if (!ffn13) {
/* 1270 */                 tfp13 = ffp13;
/* 1271 */               }
/* 1272 */
/* 1273 */               if (tfn13) {
/* 1274 */                 result.setNullAt(13);
/* 1275 */               } else {
/* 1276 */                 result.setLong(13, tfp13);
/* 1277 */               }
/* 1278 */             }
/* 1279 */
/* 1280 */
/* 1281 */             boolean ffn14 = tmpRow.isNullAt(14);
/* 1282 */             if (ffn14) {
/* 1283 */               result.setNullAt(14);
/* 1284 */             } else {
/* 1285 */               int ffp14 =
/* 1286 */               tmpRow.getInt(14);
/* 1287 */
/* 1288 */               boolean tfn14 = ffn14;
/* 1289 */               int tfp14 = -1;
/* 1290 */               if (!ffn14) {
/* 1291 */                 tfp14 = ffp14;
/* 1292 */               }
/* 1293 */
/* 1294 */               if (tfn14) {
/* 1295 */                 result.setNullAt(14);
/* 1296 */               } else {
/* 1297 */                 result.setInt(14, tfp14);
/* 1298 */               }
/* 1299 */             }
/* 1300 */
/* 1301 */
/* 1302 */             boolean ffn15 = tmpRow.isNullAt(15);
/* 1303 */             if (ffn15) {
/* 1304 */               result.setNullAt(15);
/* 1305 */             } else {
/* 1306 */               double ffp15 =
/* 1307 */               tmpRow.getDouble(15);
/* 1308 */
/* 1309 */               boolean tfn15 = ffn15;
/* 1310 */               double tfp15 = -1.0;
/* 1311 */               if (!ffn15) {
/* 1312 */                 tfp15 = ffp15;
/* 1313 */               }
/* 1314 */
/* 1315 */               if (tfn15) {
/* 1316 */                 result.setNullAt(15);
/* 1317 */               } else {
/* 1318 */                 result.setDouble(15, tfp15);
/* 1319 */               }
/* 1320 */             }
/* 1321 */
/* 1322 */
/* 1323 */             boolean ffn16 = tmpRow.isNullAt(16);
/* 1324 */             if (ffn16) {
/* 1325 */               result.setNullAt(16);
/* 1326 */             } else {
/* 1327 */               double ffp16 =
/* 1328 */               tmpRow.getDouble(16);
/* 1329 */
/* 1330 */               boolean tfn16 = ffn16;
/* 1331 */               double tfp16 = -1.0;
/* 1332 */               if (!ffn16) {
/* 1333 */                 tfp16 = ffp16;
/* 1334 */               }
/* 1335 */
/* 1336 */               if (tfn16) {
/* 1337 */                 result.setNullAt(16);
/* 1338 */               } else {
/* 1339 */                 result.setDouble(16, tfp16);
/* 1340 */               }
/* 1341 */             }
/* 1342 */
/* 1343 */
/* 1344 */             boolean ffn17 = tmpRow.isNullAt(17);
/* 1345 */             if (ffn17) {
/* 1346 */               result.setNullAt(17);
/* 1347 */             } else {
/* 1348 */               double ffp17 =
/* 1349 */               tmpRow.getDouble(17);
/* 1350 */
/* 1351 */               boolean tfn17 = ffn17;
/* 1352 */               double tfp17 = -1.0;
/* 1353 */               if (!ffn17) {
/* 1354 */                 tfp17 = ffp17;
/* 1355 */               }
/* 1356 */
/* 1357 */               if (tfn17) {
/* 1358 */                 result.setNullAt(17);
/* 1359 */               } else {
/* 1360 */                 result.setDouble(17, tfp17);
/* 1361 */               }
/* 1362 */             }
/* 1363 */
/* 1364 */
/* 1365 */             boolean ffn18 = tmpRow.isNullAt(18);
/* 1366 */             if (ffn18) {
/* 1367 */               result.setNullAt(18);
/* 1368 */             } else {
/* 1369 */               ArrayData ffp18 =
/* 1370 */               tmpRow.getArray(18);
/* 1371 */
/* 1372 */               boolean tfn18 = ffn18;
/* 1373 */               ArrayData tfp18 = null;
/* 1374 */               if (!ffn18) {
/* 1375 */                 tfp18 = ffp18;
/* 1376 */               }
/* 1377 */
/* 1378 */               if (tfn18) {
/* 1379 */                 result.setNullAt(18);
/* 1380 */               } else {
/* 1381 */                 result.update(18, tfp18);
/* 1382 */               }
/* 1383 */             }
/* 1384 */
/* 1385 */
/* 1386 */             boolean ffn19 = tmpRow.isNullAt(19);
/* 1387 */             if (ffn19) {
/* 1388 */               result.setNullAt(19);
/* 1389 */             } else {
/* 1390 */               ArrayData ffp19 =
/* 1391 */               tmpRow.getArray(19);
/* 1392 */
/* 1393 */               boolean tfn19 = ffn19;
/* 1394 */               ArrayData tfp19 = null;
/* 1395 */               if (!ffn19) {
/* 1396 */                 tfp19 = ffp19;
/* 1397 */               }
/* 1398 */
/* 1399 */               if (tfn19) {
/* 1400 */                 result.setNullAt(19);
/* 1401 */               } else {
/* 1402 */                 result.update(19, tfp19);
/* 1403 */               }
/* 1404 */             }
/* 1405 */
/* 1406 */
/* 1407 */             boolean ffn20 = tmpRow.isNullAt(20);
/* 1408 */             if (ffn20) {
/* 1409 */               result.setNullAt(20);
/* 1410 */             } else {
/* 1411 */               ArrayData ffp20 =
/* 1412 */               tmpRow.getArray(20);
/* 1413 */
/* 1414 */               boolean tfn20 = ffn20;
/* 1415 */               ArrayData tfp20 = null;
/* 1416 */               if (!ffn20) {
/* 1417 */                 tfp20 = ffp20;
/* 1418 */               }
/* 1419 */
/* 1420 */               if (tfn20) {
/* 1421 */                 result.setNullAt(20);
/* 1422 */               } else {
/* 1423 */                 result.update(20, tfp20);
/* 1424 */               }
/* 1425 */             }
/* 1426 */
/* 1427 */             tePrim = result.copy();
/* 1428 */
/* 1429 */           }
/* 1430 */
/* 1431 */           if (teNull) {
/* 1432 */             values[j] = null;
/* 1433 */           } else {
/* 1434 */             values[j] = tePrim;
/* 1435 */           }
/* 1436 */         }
/* 1437 */       }
/* 1438 */       value166 = new org.apache.spark.sql.catalyst.util.GenericArrayData(values);
/* 1439 */
/* 1440 */     }
/* 1441 */     ArrayData value165 = null;
/* 1442 */
/* 1443 */     if (!isNull166) {
/* 1444 */
/* 1445 */       indexer.dto.ticket.VendorItemDto[] convertedArray19 = null;
/* 1446 */       int dataLength19 = value166.numElements();
/* 1447 */       convertedArray19 = new indexer.dto.ticket.VendorItemDto[dataLength19];
/* 1448 */
/* 1449 */       int loopIndex19 = 0;
/* 1450 */       while (loopIndex19 &lt; dataLength19) {
/* 1451 */         MapObjects_loopValue378 = (InternalRow) (value166.getStruct(loopIndex19, 21));
/* 1452 */         MapObjects_loopIsNull379 = value166.isNullAt(loopIndex19);
/* 1453 */
/* 1454 */
/* 1455 */         boolean isNull168 = false;
/* 1456 */         indexer.dto.ticket.VendorItemDto value168 = null;
/* 1457 */         if (!false &amp;&amp; MapObjects_loopIsNull379) {
/* 1458 */
/* 1459 */           final indexer.dto.ticket.VendorItemDto value170 = null;
/* 1460 */           isNull168 = true;
/* 1461 */           value168 = value170;
/* 1462 */         } else {
/* 1463 */
/* 1464 */           apply1_0(i);
/* 1465 */           apply1_1(i);
/* 1466 */           apply1_2(i);
/* 1467 */           apply1_3(i);
/* 1468 */           apply1_4(i);
/* 1469 */           apply1_5(i);
/* 1470 */           apply1_6(i);
/* 1471 */           apply1_7(i);
/* 1472 */           apply1_8(i);
/* 1473 */           apply1_9(i);
/* 1474 */           apply1_10(i);
/* 1475 */           apply1_11(i);
/* 1476 */           apply1_12(i);
/* 1477 */
/* 1478 */           final indexer.dto.ticket.VendorItemDto value171 = false ? null : new indexer.dto.ticket.VendorItemDto(argValue92, argValue93, argValue94, argValue95, argValue96, argValue97, argValue98, argValue99, argValue100, argValue101, argValue102, argValue103, argValue104, argValue105, argValue106, argValue107, argValue108, argValue109, argValue110, argValue111, argValue112);
/* 1479 */           isNull168 = false;
/* 1480 */           value168 = value171;
/* 1481 */         }
/* 1482 */         if (isNull168) {
/* 1483 */           convertedArray19[loopIndex19] = null;
/* 1484 */         } else {
/* 1485 */           convertedArray19[loopIndex19] = value168;
/* 1486 */         }
/* 1487 */
/* 1488 */         loopIndex19 += 1;
/* 1489 */       }
/* 1490 */
/* 1491 */       value165 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray19);
/* 1492 */     }
/* 1493 */     boolean isNull164 = true;
/* 1494 */     indexer.dto.ticket.VendorItemDto[] value164 = null;
/* 1495 */     if (!isNull166) {
/* 1496 */
/* 1497 */       isNull164 = false;
/* 1498 */       if (!isNull164) {
/* 1499 */
/* 1500 */         Object funcResult56 = null;
/* 1501 */         funcResult56 = value165.array();
/* 1502 */         if (funcResult56 == null) {
/* 1503 */           isNull164 = true;
/* 1504 */         } else {
/* 1505 */           value164 = (indexer.dto.ticket.VendorItemDto[]) funcResult56;
/* 1506 */         }
/* 1507 */
/* 1508 */       }
/* 1509 */       isNull164 = value164 == null;
/* 1510 */     }
/* 1511 */     argValue37 = value164;
/* 1512 */
/* 1513 */   }


Thanks a lot
Proust



地址: 上海市浦东新区金科路2889弄长泰广场C座7楼 
Address: 7th floor, No#3, Jinke road 2889, Pudong district, Shanghai, China.
Mobile: +86 13621672634