Getting error message

classic Classic list List threaded Threaded
7 messages Options
Reply | Threaded
Open this post in threaded view
|

Getting error message

sperry.it@gmail.com
Hi,

Since last few days, I am getting error message while running my project. I have searched Google for the solution but didn't got any help.

Can someone help me to figure out how I could mitigate this issue?


20/12/17 13:26:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

        at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

        at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

        at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

        at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

        at scala.Option.orElse(Option.scala:447)

        at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

        at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)

        at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR TaskSetManager: Task 1 in stage 39.0 failed 1 times; aborting job

20/12/17 13:27:12 ERROR FileFormatWriter: Aborting job 0db8a16a-e1d3-4091-b725-0a8bdc17b0b1.

java.util.concurrent.ExecutionException: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times, most recent failure: Lost task 1.0 in stage 39.0 (TID 655, my-domain.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

        at java.util.concurrent.FutureTask.report(FutureTask.java:122)

        at java.util.concurrent.FutureTask.get(FutureTask.java:206)

        at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(BroadcastExchangeExec.scala:195)

        at org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(WholeStageCodegenExec.scala:515)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:188)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:184)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(BroadcastHashJoinExec.scala:116)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenInner(BroadcastHashJoinExec.scala:210)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:100)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.FilterExec.consume(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.FilterExec.doConsume(basicPhysicalOperators.scala:222)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ColumnarToRowExec.consume(Columnar.scala:60)

        at org.apache.spark.sql.execution.ColumnarToRowExec.doProduce(Columnar.scala:185)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ColumnarToRowExec.produce(Columnar.scala:60)

        at org.apache.spark.sql.execution.FilterExec.doProduce(basicPhysicalOperators.scala:137)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.FilterExec.produce(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(WholeStageCodegenExec.scala:632)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:692)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

        at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:172)

        at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:178)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

        at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:122)

        at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:121)

        at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:963)

        at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)

        at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)

        at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)

        at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)

        at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)

        at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:963)

        at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:415)

        at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:399)

        at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:288)

        at org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:848)

        at com.brighthealthplan.qnxt.sparkjobs.DataManager.normalizeSource(DataManager.scala:190)

        at com.brighthealthplan.qnxt.sparkjobs.Main$.main(Main.scala:100)

        at com.brighthealthplan.qnxt.sparkjobs.Main.main(Main.scala)

Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times, most recent failure: Lost task 1.0 in stage 39.0 (TID 655, my-domain.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

        at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2059)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2008)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2007)

        at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)

        at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)

        at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)

        at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2007)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:973)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:973)

        at scala.Option.foreach(Option.scala:407)

        at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:973)

        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2239)

        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2188)

        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2177)

        at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)

        at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:775)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2099)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2120)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2139)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2164)

        at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1004)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)

        at org.apache.spark.rdd.RDD.withScope(RDD.scala:388)

        at org.apache.spark.rdd.RDD.collect(RDD.scala:1003)

        at org.apache.spark.sql.execution.SparkPlan.executeCollectIterator(SparkPlan.scala:392)

        at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.$anonfun$relationFuture$1(BroadcastExchangeExec.scala:120)

        at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:182)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at java.lang.Thread.run(Thread.java:748)

Exception in thread "main" org.apache.spark.SparkException: Job aborted.

        at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:226)

        at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:178)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

        at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:122)

        at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:121)

        at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:963)

        at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)

        at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)

        at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)

        at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)

        at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)

        at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:963)

        at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:415)

        at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:399)

        at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:288)

        at org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:848)

        at com.brighthealthplan.qnxt.sparkjobs.DataManager.normalizeSource(DataManager.scala:190)

        at com.brighthealthplan.qnxt.sparkjobs.Main$.main(Main.scala:100)

        at com.brighthealthplan.qnxt.sparkjobs.Main.main(Main.scala)

Caused by: java.util.concurrent.ExecutionException: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times, most recent failure: Lost task 1.0 in stage 39.0 (TID 655, my-domain.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

        at java.util.concurrent.FutureTask.report(FutureTask.java:122)

        at java.util.concurrent.FutureTask.get(FutureTask.java:206)

        at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(BroadcastExchangeExec.scala:195)

        at org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(WholeStageCodegenExec.scala:515)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:188)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:184)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(BroadcastHashJoinExec.scala:116)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenInner(BroadcastHashJoinExec.scala:210)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:100)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.FilterExec.consume(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.FilterExec.doConsume(basicPhysicalOperators.scala:222)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ColumnarToRowExec.consume(Columnar.scala:60)

        at org.apache.spark.sql.execution.ColumnarToRowExec.doProduce(Columnar.scala:185)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ColumnarToRowExec.produce(Columnar.scala:60)

        at org.apache.spark.sql.execution.FilterExec.doProduce(basicPhysicalOperators.scala:137)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.FilterExec.produce(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(WholeStageCodegenExec.scala:632)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:692)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

        at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:172)

        ... 25 more

Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times, most recent failure: Lost task 1.0 in stage 39.0 (TID 655, my-domain.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

        at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2059)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2008)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2007)

        at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)

        at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)

        at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)

        at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2007)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:973)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:973)

        at scala.Option.foreach(Option.scala:407)

        at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:973)

        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2239)

        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2188)

        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2177)

        at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)

        at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:775)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2099)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2120)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2139)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2164)

        at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1004)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)

        at org.apache.spark.rdd.RDD.withScope(RDD.scala:388)

        at org.apache.spark.rdd.RDD.collect(RDD.scala:1003)

        at org.apache.spark.sql.execution.SparkPlan.executeCollectIterator(SparkPlan.scala:392)

        at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.$anonfun$relationFuture$1(BroadcastExchangeExec.scala:120)

        at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:182)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at java.lang.Thread.run(Thread.java:748)

 

Reply | Threaded
Open this post in threaded view
|

Re: Getting error message

Patrick McCarthy-2
my-domain.com/192.168.166.8:63534 probably isn't a valid address on your network, is it?

On Thu, Dec 17, 2020 at 3:03 AM Vikas Garg <[hidden email]> wrote:
Hi,

Since last few days, I am getting error message while running my project. I have searched Google for the solution but didn't got any help.

Can someone help me to figure out how I could mitigate this issue?


20/12/17 13:26:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

        at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

        at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

        at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

        at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

        at scala.Option.orElse(Option.scala:447)

        at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

        at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)

        at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR TaskSetManager: Task 1 in stage 39.0 failed 1 times; aborting job

20/12/17 13:27:12 ERROR FileFormatWriter: Aborting job 0db8a16a-e1d3-4091-b725-0a8bdc17b0b1.

java.util.concurrent.ExecutionException: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times, most recent failure: Lost task 1.0 in stage 39.0 (TID 655, my-domain.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

        at java.util.concurrent.FutureTask.report(FutureTask.java:122)

        at java.util.concurrent.FutureTask.get(FutureTask.java:206)

        at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(BroadcastExchangeExec.scala:195)

        at org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(WholeStageCodegenExec.scala:515)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:188)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:184)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(BroadcastHashJoinExec.scala:116)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenInner(BroadcastHashJoinExec.scala:210)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:100)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.FilterExec.consume(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.FilterExec.doConsume(basicPhysicalOperators.scala:222)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ColumnarToRowExec.consume(Columnar.scala:60)

        at org.apache.spark.sql.execution.ColumnarToRowExec.doProduce(Columnar.scala:185)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ColumnarToRowExec.produce(Columnar.scala:60)

        at org.apache.spark.sql.execution.FilterExec.doProduce(basicPhysicalOperators.scala:137)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.FilterExec.produce(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(WholeStageCodegenExec.scala:632)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:692)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

        at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:172)

        at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:178)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

        at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:122)

        at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:121)

        at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:963)

        at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)

        at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)

        at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)

        at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)

        at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)

        at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:963)

        at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:415)

        at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:399)

        at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:288)

        at org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:848)

        at com.brighthealthplan.qnxt.sparkjobs.DataManager.normalizeSource(DataManager.scala:190)

        at com.brighthealthplan.qnxt.sparkjobs.Main$.main(Main.scala:100)

        at com.brighthealthplan.qnxt.sparkjobs.Main.main(Main.scala)

Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times, most recent failure: Lost task 1.0 in stage 39.0 (TID 655, my-domain.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

        at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2059)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2008)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2007)

        at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)

        at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)

        at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)

        at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2007)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:973)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:973)

        at scala.Option.foreach(Option.scala:407)

        at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:973)

        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2239)

        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2188)

        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2177)

        at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)

        at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:775)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2099)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2120)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2139)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2164)

        at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1004)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)

        at org.apache.spark.rdd.RDD.withScope(RDD.scala:388)

        at org.apache.spark.rdd.RDD.collect(RDD.scala:1003)

        at org.apache.spark.sql.execution.SparkPlan.executeCollectIterator(SparkPlan.scala:392)

        at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.$anonfun$relationFuture$1(BroadcastExchangeExec.scala:120)

        at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:182)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at java.lang.Thread.run(Thread.java:748)

Exception in thread "main" org.apache.spark.SparkException: Job aborted.

        at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:226)

        at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:178)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

        at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:122)

        at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:121)

        at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:963)

        at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)

        at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)

        at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)

        at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)

        at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)

        at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:963)

        at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:415)

        at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:399)

        at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:288)

        at org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:848)

        at com.brighthealthplan.qnxt.sparkjobs.DataManager.normalizeSource(DataManager.scala:190)

        at com.brighthealthplan.qnxt.sparkjobs.Main$.main(Main.scala:100)

        at com.brighthealthplan.qnxt.sparkjobs.Main.main(Main.scala)

Caused by: java.util.concurrent.ExecutionException: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times, most recent failure: Lost task 1.0 in stage 39.0 (TID 655, my-domain.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

        at java.util.concurrent.FutureTask.report(FutureTask.java:122)

        at java.util.concurrent.FutureTask.get(FutureTask.java:206)

        at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(BroadcastExchangeExec.scala:195)

        at org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(WholeStageCodegenExec.scala:515)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:188)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:184)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(BroadcastHashJoinExec.scala:116)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenInner(BroadcastHashJoinExec.scala:210)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:100)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.FilterExec.consume(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.FilterExec.doConsume(basicPhysicalOperators.scala:222)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ColumnarToRowExec.consume(Columnar.scala:60)

        at org.apache.spark.sql.execution.ColumnarToRowExec.doProduce(Columnar.scala:185)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ColumnarToRowExec.produce(Columnar.scala:60)

        at org.apache.spark.sql.execution.FilterExec.doProduce(basicPhysicalOperators.scala:137)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.FilterExec.produce(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(WholeStageCodegenExec.scala:632)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:692)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

        at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:172)

        ... 25 more

Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times, most recent failure: Lost task 1.0 in stage 39.0 (TID 655, my-domain.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

        at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2059)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2008)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2007)

        at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)

        at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)

        at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)

        at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2007)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:973)

        at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:973)

        at scala.Option.foreach(Option.scala:407)

        at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:973)

        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2239)

        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2188)

        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2177)

        at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)

        at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:775)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2099)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2120)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2139)

        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2164)

        at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1004)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)

        at org.apache.spark.rdd.RDD.withScope(RDD.scala:388)

        at org.apache.spark.rdd.RDD.collect(RDD.scala:1003)

        at org.apache.spark.sql.execution.SparkPlan.executeCollectIterator(SparkPlan.scala:392)

        at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.$anonfun$relationFuture$1(BroadcastExchangeExec.scala:120)

        at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:182)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at java.lang.Thread.run(Thread.java:748)

 



--

Patrick McCarthy 

Senior Data Scientist, Machine Learning Engineering

Dstillery

470 Park Ave South, 17th Floor, NYC 10016

Reply | Threaded
Open this post in threaded view
|

Re: Getting error message

sperry.it@gmail.com
Mydomain is named by me while pasting the logs

Also,  there are multiple class files in my project, if I run any 1 or 2 at a time,  then they run fine,  sometimes they too give this error. But running all the classes at the same time always give this error. 

Once this error come, I can't run any program and on restarting the system, program starts running fine. 
This error goes away on

On Thu, 17 Dec 2020, 23:50 Patrick McCarthy, <[hidden email]> wrote:
my-domain.com/192.168.166.8:63534 probably isn't a valid address on your network, is it?

On Thu, Dec 17, 2020 at 3:03 AM Vikas Garg <[hidden email]> wrote:
Hi,

Since last few days, I am getting error message while running my project. I have searched Google for the solution but didn't got any help.

Can someone help me to figure out how I could mitigate this issue?


20/12/17 13:26:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

        at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

        at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

        at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

        at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

        at scala.Option.orElse(Option.scala:447)

        at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

        at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)

        at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR TaskSetManager: Task 1 in stage 39.0 failed 1 times; aborting job

20/12/17 13:27:12 ERROR FileFormatWriter: Aborting job 0db8a16a-e1d3-4091-b725-0a8bdc17b0b1.

java.util.concurrent.ExecutionException: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times, most recent failure: Lost task 1.0 in stage 39.0 (TID 655, my-domain.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

        at java.util.concurrent.FutureTask.report(FutureTask.java:122)

        at java.util.concurrent.FutureTask.get(FutureTask.java:206)

        at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(BroadcastExchangeExec.scala:195)

        at org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(WholeStageCodegenExec.scala:515)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:188)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:184)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(BroadcastHashJoinExec.scala:116)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenInner(BroadcastHashJoinExec.scala:210)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:100)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.FilterExec.consume(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.FilterExec.doConsume(basicPhysicalOperators.scala:222)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ColumnarToRowExec.consume(Columnar.scala:60)

        at org.apache.spark.sql.execution.ColumnarToRowExec.doProduce(Columnar.scala:185)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ColumnarToRowExec.produce(Columnar.scala:60)

        at org.apache.spark.sql.execution.FilterExec.doProduce(basicPhysicalOperators.scala:137)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.FilterExec.produce(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(WholeStageCodegenExec.scala:632)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:692)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

        at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:172)

        at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:178)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execu

Reply | Threaded
Open this post in threaded view
|

Re: Getting error message

Patrick McCarthy-2
'Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times'

You may want to change the number of failures to a higher number like 4. A single failure on a task should be able to be tolerated, especially if you're on a shared cluster where resources can be preempted.

 It seems that a node dies or goes off the network, so perhaps you can also debug the logs on the failing node to see why it disappears and prevent the failures in the first place.

On Thu, Dec 17, 2020 at 1:27 PM Vikas Garg <[hidden email]> wrote:
Mydomain is named by me while pasting the logs

Also,  there are multiple class files in my project, if I run any 1 or 2 at a time,  then they run fine,  sometimes they too give this error. But running all the classes at the same time always give this error. 

Once this error come, I can't run any program and on restarting the system, program starts running fine. 
This error goes away on

On Thu, 17 Dec 2020, 23:50 Patrick McCarthy, <[hidden email]> wrote:
my-domain.com/192.168.166.8:63534 probably isn't a valid address on your network, is it?

On Thu, Dec 17, 2020 at 3:03 AM Vikas Garg <[hidden email]> wrote:
Hi,

Since last few days, I am getting error message while running my project. I have searched Google for the solution but didn't got any help.

Can someone help me to figure out how I could mitigate this issue?


20/12/17 13:26:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

        at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

        at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

        at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

        at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

        at scala.Option.orElse(Option.scala:447)

        at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

        at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)

        at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR TaskSetManager: Task 1 in stage 39.0 failed 1 times; aborting job

20/12/17 13:27:12 ERROR FileFormatWriter: Aborting job 0db8a16a-e1d3-4091-b725-0a8bdc17b0b1.

java.util.concurrent.ExecutionException: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times, most recent failure: Lost task 1.0 in stage 39.0 (TID 655, my-domain.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

        at java.util.concurrent.FutureTask.report(FutureTask.java:122)

        at java.util.concurrent.FutureTask.get(FutureTask.java:206)

        at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(BroadcastExchangeExec.scala:195)

        at org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(WholeStageCodegenExec.scala:515)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:188)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:184)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(BroadcastHashJoinExec.scala:116)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenInner(BroadcastHashJoinExec.scala:210)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:100)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.FilterExec.consume(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.FilterExec.doConsume(basicPhysicalOperators.scala:222)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ColumnarToRowExec.consume(Columnar.scala:60)

        at org.apache.spark.sql.execution.ColumnarToRowExec.doProduce(Columnar.scala:185)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ColumnarToRowExec.produce(Columnar.scala:60)

        at org.apache.spark.sql.execution.FilterExec.doProduce(basicPhysicalOperators.scala:137)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.FilterExec.produce(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(WholeStageCodegenExec.scala:632)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:692)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

        at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:172)

        at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:178)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execu



--

Patrick McCarthy 

Senior Data Scientist, Machine Learning Engineering

Dstillery

470 Park Ave South, 17th Floor, NYC 10016

Reply | Threaded
Open this post in threaded view
|

Re: Getting error message

sperry.it@gmail.com
I am running code in a local machine that is single node machine. 

Getting into logs,  it looked like the host is killed.  This is happening very frequently an I am unable to find the reason of this.

Could low memory be the reason? 

On Fri, 18 Dec 2020, 00:11 Patrick McCarthy, <[hidden email]> wrote:
'Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times'

You may want to change the number of failures to a higher number like 4. A single failure on a task should be able to be tolerated, especially if you're on a shared cluster where resources can be preempted.

 It seems that a node dies or goes off the network, so perhaps you can also debug the logs on the failing node to see why it disappears and prevent the failures in the first place.

On Thu, Dec 17, 2020 at 1:27 PM Vikas Garg <[hidden email]> wrote:
Mydomain is named by me while pasting the logs

Also,  there are multiple class files in my project, if I run any 1 or 2 at a time,  then they run fine,  sometimes they too give this error. But running all the classes at the same time always give this error. 

Once this error come, I can't run any program and on restarting the system, program starts running fine. 
This error goes away on

On Thu, 17 Dec 2020, 23:50 Patrick McCarthy, <[hidden email]> wrote:
my-domain.com/192.168.166.8:63534 probably isn't a valid address on your network, is it?

On Thu, Dec 17, 2020 at 3:03 AM Vikas Garg <[hidden email]> wrote:
Hi,

Since last few days, I am getting error message while running my project. I have searched Google for the solution but didn't got any help.

Can someone help me to figure out how I could mitigate this issue?


20/12/17 13:26:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

        at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

        at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

        at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

        at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

        at scala.Option.orElse(Option.scala:447)

        at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

        at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)

        at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR TaskSetManager: Task 1 in stage 39.0 failed 1 times; aborting job

20/12/17 13:27:12 ERROR FileFormatWriter: Aborting job 0db8a16a-e1d3-4091-b725-0a8bdc17b0b1.

java.util.concurrent.ExecutionException: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times, most recent failure: Lost task 1.0 in stage 39.0 (TID 655, my-domain.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

        at java.util.concurrent.FutureTask.report(FutureTask.java:122)

        at java.util.concurrent.FutureTask.get(FutureTask.java:206)

        at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(BroadcastExchangeExec.scala:195)

        at org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(WholeStageCodegenExec.scala:515)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:188)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:184)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(BroadcastHashJoinExec.scala:116)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenInner(BroadcastHashJoinExec.scala:210)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:100)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.FilterExec.consume(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.FilterExec.doConsume(basicPhysicalOperators.scala:222)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ColumnarToRowExec.consume(Columnar.scala:60)

        at org.apache.spark.sql.execution.ColumnarToRowExec.doProduce(Columnar.scala:185)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ColumnarToRowExec.produce(Columnar.scala:60)

        at org.apache.spark.sql.execution.FilterExec.doProduce(basicPhysicalOperators.scala:137)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.FilterExec.produce(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(WholeStageCodegenExec.scala:632)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:692)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

        at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:172)

        at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:178)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execu



--

Patrick McCarthy 

Senior Data Scientist, Machine Learning Engineering

Dstillery

470 Park Ave South, 17th Floor, NYC 10016

Reply | Threaded
Open this post in threaded view
|

Re: Getting error message

Patrick McCarthy-2
Possibly. In that case maybe you should step back from spark and see if there are OS-level tools to understand what's going on, like looking for evidence of the OOM killer - https://docs.memset.com/other/linux-s-oom-process-killer 

On Thu, Dec 17, 2020 at 1:45 PM Vikas Garg <[hidden email]> wrote:
I am running code in a local machine that is single node machine. 

Getting into logs,  it looked like the host is killed.  This is happening very frequently an I am unable to find the reason of this.

Could low memory be the reason? 

On Fri, 18 Dec 2020, 00:11 Patrick McCarthy, <[hidden email]> wrote:
'Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times'

You may want to change the number of failures to a higher number like 4. A single failure on a task should be able to be tolerated, especially if you're on a shared cluster where resources can be preempted.

 It seems that a node dies or goes off the network, so perhaps you can also debug the logs on the failing node to see why it disappears and prevent the failures in the first place.

On Thu, Dec 17, 2020 at 1:27 PM Vikas Garg <[hidden email]> wrote:
Mydomain is named by me while pasting the logs

Also,  there are multiple class files in my project, if I run any 1 or 2 at a time,  then they run fine,  sometimes they too give this error. But running all the classes at the same time always give this error. 

Once this error come, I can't run any program and on restarting the system, program starts running fine. 
This error goes away on

On Thu, 17 Dec 2020, 23:50 Patrick McCarthy, <[hidden email]> wrote:
my-domain.com/192.168.166.8:63534 probably isn't a valid address on your network, is it?

On Thu, Dec 17, 2020 at 3:03 AM Vikas Garg <[hidden email]> wrote:
Hi,

Since last few days, I am getting error message while running my project. I have searched Google for the solution but didn't got any help.

Can someone help me to figure out how I could mitigate this issue?


20/12/17 13:26:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

        at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

        at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

        at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

        at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

        at scala.Option.orElse(Option.scala:447)

        at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

        at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)

        at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR TaskSetManager: Task 1 in stage 39.0 failed 1 times; aborting job

20/12/17 13:27:12 ERROR FileFormatWriter: Aborting job 0db8a16a-e1d3-4091-b725-0a8bdc17b0b1.

java.util.concurrent.ExecutionException: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times, most recent failure: Lost task 1.0 in stage 39.0 (TID 655, my-domain.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

        at java.util.concurrent.FutureTask.report(FutureTask.java:122)

        at java.util.concurrent.FutureTask.get(FutureTask.java:206)

        at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(BroadcastExchangeExec.scala:195)

        at org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(WholeStageCodegenExec.scala:515)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:188)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:184)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(BroadcastHashJoinExec.scala:116)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenInner(BroadcastHashJoinExec.scala:210)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:100)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.FilterExec.consume(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.FilterExec.doConsume(basicPhysicalOperators.scala:222)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ColumnarToRowExec.consume(Columnar.scala:60)

        at org.apache.spark.sql.execution.ColumnarToRowExec.doProduce(Columnar.scala:185)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ColumnarToRowExec.produce(Columnar.scala:60)

        at org.apache.spark.sql.execution.FilterExec.doProduce(basicPhysicalOperators.scala:137)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.FilterExec.produce(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(WholeStageCodegenExec.scala:632)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:692)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

        at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:172)

        at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:178)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)

        at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execu



--

Patrick McCarthy 

Senior Data Scientist, Machine Learning Engineering

Dstillery

470 Park Ave South, 17th Floor, NYC 10016



--

Patrick McCarthy 

Senior Data Scientist, Machine Learning Engineering

Dstillery

470 Park Ave South, 17th Floor, NYC 10016

Reply | Threaded
Open this post in threaded view
|

Re: Getting error message

sperry.it@gmail.com
Thanks 

On Fri, 18 Dec 2020, 00:30 Patrick McCarthy, <[hidden email]> wrote:
Possibly. In that case maybe you should step back from spark and see if there are OS-level tools to understand what's going on, like looking for evidence of the OOM killer - https://docs.memset.com/other/linux-s-oom-process-killer 

On Thu, Dec 17, 2020 at 1:45 PM Vikas Garg <[hidden email]> wrote:
I am running code in a local machine that is single node machine. 

Getting into logs,  it looked like the host is killed.  This is happening very frequently an I am unable to find the reason of this.

Could low memory be the reason? 

On Fri, 18 Dec 2020, 00:11 Patrick McCarthy, <[hidden email]> wrote:
'Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times'

You may want to change the number of failures to a higher number like 4. A single failure on a task should be able to be tolerated, especially if you're on a shared cluster where resources can be preempted.

 It seems that a node dies or goes off the network, so perhaps you can also debug the logs on the failing node to see why it disappears and prevent the failures in the first place.

On Thu, Dec 17, 2020 at 1:27 PM Vikas Garg <[hidden email]> wrote:
Mydomain is named by me while pasting the logs

Also,  there are multiple class files in my project, if I run any 1 or 2 at a time,  then they run fine,  sometimes they too give this error. But running all the classes at the same time always give this error. 

Once this error come, I can't run any program and on restarting the system, program starts running fine. 
This error goes away on

On Thu, 17 Dec 2020, 23:50 Patrick McCarthy, <[hidden email]> wrote:
my-domain.com/192.168.166.8:63534 probably isn't a valid address on your network, is it?

On Thu, Dec 17, 2020 at 3:03 AM Vikas Garg <[hidden email]> wrote:
Hi,

Since last few days, I am getting error message while running my project. I have searched Google for the solution but didn't got any help.

Can someone help me to figure out how I could mitigate this issue?


20/12/17 13:26:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

        at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

        at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

        at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

        at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

        at scala.Option.orElse(Option.scala:447)

        at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

        at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)

        at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to my-domain.com/192.168.166.8:63534

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

        at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

        at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

        at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

        at java.util.concurrent.FutureTask.run(FutureTask.java:266)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: my-domain.com/192.168.166.8:63534

Caused by: java.net.SocketException: Permission denied: no further information

        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

        at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

        at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

        at java.lang.Thread.run(Thread.java:748)

20/12/17 13:27:12 ERROR TaskSetManager: Task 1 in stage 39.0 failed 1 times; aborting job

20/12/17 13:27:12 ERROR FileFormatWriter: Aborting job 0db8a16a-e1d3-4091-b725-0a8bdc17b0b1.

java.util.concurrent.ExecutionException: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 39.0 failed 1 times, most recent failure: Lost task 1.0 in stage 39.0 (TID 655, my-domain.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

        at java.util.concurrent.FutureTask.report(FutureTask.java:122)

        at java.util.concurrent.FutureTask.get(FutureTask.java:206)

        at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(BroadcastExchangeExec.scala:195)

        at org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(WholeStageCodegenExec.scala:515)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:188)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:184)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(BroadcastHashJoinExec.scala:116)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenInner(BroadcastHashJoinExec.scala:210)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:100)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.FilterExec.consume(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.FilterExec.doConsume(basicPhysicalOperators.scala:222)

        at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

        at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

        at org.apache.spark.sql.execution.ColumnarToRowExec.consume(Columnar.scala:60)

        at org.apache.spark.sql.execution.ColumnarToRowExec.doProduce(Columnar.scala:185)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ColumnarToRowExec.produce(Columnar.scala:60)

        at org.apache.spark.sql.execution.FilterExec.doProduce(basicPhysicalOperators.scala:137)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.FilterExec.produce(basicPhysicalOperators.scala:97)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

        at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

        at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

        at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(WholeStageCodegenExec.scala:632)

        at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:692)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

        at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

        at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

        at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

        at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:172)