Issue with executer

classic Classic list List threaded Threaded
6 messages Options
Reply | Threaded
Open this post in threaded view
|

Issue with executer

sperry.it@gmail.com
Hi,

I am facing issue with spark executor. I am struggling with this issue since last many days and unable to resolve the issue.

Below is the configuration I have given.

  val spark = SparkSession.builder()
    .appName("Spark Job")
    .master("local[*]")
    .config("spark.dynamicAllocation.enabled", true)
    .config("spark.shuffle.service.enabled", true)
    .config("spark.driver.maxResultSize", "8g")
    .config("spark.driver.memory", "8g")
    .config("spark.executor.memory", "8g")
    .config("spark.network.timeout", "3600s")
    .getOrCreate()

1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

            at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

            at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

            at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

            at scala.Option.orElse(Option.scala:289)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

            at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

            at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

            at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

            at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

            at scala.Option.orElse(Option.scala:289)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

            at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:12 ERROR TaskSetManager: Task 0 in stage 23.0 failed 1 times; aborting job

21/01/20 17:07:12 ERROR TaskSetManager: Task 2 in stage 17.0 failed 1 times; aborting job

Exception in thread "main" org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree:

Exchange hashpartitioning(claimid#1012, 200), true, [id=#742]

 

 

            at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56)

            at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.doExecute(ShuffleExchangeExec.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

            at org.apache.spark.sql.execution.InputAdapter.inputRDD(WholeStageCodegenExec.scala:525)

            at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(WholeStageCodegenExec.scala:453)

            at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(WholeStageCodegenExec.scala:452)

            at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:496)

            at org.apache.spark.sql.execution.SortExec.inputRDDs(SortExec.scala:132)

            at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:720)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

            at org.apache.spark.sql.execution.joins.SortMergeJoinExec.doExecute(SortMergeJoinExec.scala:182)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

            at org.apache.spark.sql.execution.ProjectExec.doExecute(basicPhysicalOperators.scala:76)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

            at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:316)

            at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:434)

            at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:420)

            at org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:47)

            at org.apache.spark.sql.Dataset.collectFromPlan(Dataset.scala:3627)

            at org.apache.spark.sql.Dataset.$anonfun$head$1(Dataset.scala:2697)

            at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3618)

            at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)

            at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)

            at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)

            at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)

            at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)

            at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3616)

            at org.apache.spark.sql.Dataset.head(Dataset.scala:2697)

            at org.apache.spark.sql.Dataset.take(Dataset.scala:2904)

            at org.apache.spark.sql.Dataset.getRows(Dataset.scala:300)

            at org.apache.spark.sql.Dataset.showString(Dataset.scala:337)

            at org.apache.spark.sql.Dataset.show(Dataset.scala:824)

            at org.apache.spark.sql.Dataset.show(Dataset.scala:783)

            at org.apache.spark.sql.Dataset.show(Dataset.scala:792)

            at com.brighthealthplan.cognizant.output.MedicalClaim$.getDS(MedicalClaim.scala:257)

            at com.brighthealthplan.cognizant.sparkjobs.DataManager.writeLeft(DataManager.scala:192)

            at com.brighthealthplan.cognizant.sparkjobs.Main$.main(Main.scala:107)

            at com.brighthealthplan.cognizant.sparkjobs.Main.main(Main.scala)

Caused by: java.util.concurrent.ExecutionException: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 23.0 failed 1 times, most recent failure: Lost task 0.0 in stage 23.0 (TID 29, del1-lhp-n99999.synapse.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

            at java.util.concurrent.FutureTask.report(FutureTask.java:122)

            at java.util.concurrent.FutureTask.get(FutureTask.java:206)

            at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(BroadcastExchangeExec.scala:195)

            at org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(WholeStageCodegenExec.scala:515)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:188)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:184)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(BroadcastHashJoinExec.scala:116)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(BroadcastHashJoinExec.scala:258)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:101)

            at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

            at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

            at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.consume(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(BroadcastHashJoinExec.scala:302)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:101)

            at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

            at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

            at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.consume(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(BroadcastHashJoinExec.scala:302)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:101)

            at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

            at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

            at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

            at org.apache.spark.sql.execution.InputAdapter.consume(WholeStageCodegenExec.scala:496)

            at org.apache.spark.sql.execution.InputRDDCodegen.doProduce(WholeStageCodegenExec.scala:483)

            at org.apache.spark.sql.execution.InputRDDCodegen.doProduce$(WholeStageCodegenExec.scala:456)

            at org.apache.spark.sql.execution.InputAdapter.doProduce(WholeStageCodegenExec.scala:496)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.InputAdapter.produce(WholeStageCodegenExec.scala:496)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(WholeStageCodegenExec.scala:632)

            at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:692)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

            at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD$lzycompute(ShuffleExchangeExec.scala:106)

            at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD(ShuffleExchangeExec.scala:106)

            at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency$lzycompute(ShuffleExchangeExec.scala:139)

            at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency(ShuffleExchangeExec.scala:137)

            at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.$anonfun$doExecute$1(ShuffleExchangeExec.scala:154)

            at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52)

            ... 53 more

Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 23.0 failed 1 times, most recent failure: Lost task 0.0 in stage 23.0 (TID 29, del1-lhp-n99999.synapse.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

            at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2059)

            at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2008)

            at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2007)

            at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:59)

            at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:52)

            at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)

            at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2007)

            at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:973)

            at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:973)

            at scala.Option.foreach(Option.scala:257)

            at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:973)

            at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2239)

            at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2188)

            at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2177)

            at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)

            at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:775)

            at org.apache.spark.SparkContext.runJob(SparkContext.scala:2099)

            at org.apache.spark.SparkContext.runJob(SparkContext.scala:2120)

            at org.apache.spark.SparkContext.runJob(SparkContext.scala:2139)

            at org.apache.spark.SparkContext.runJob(SparkContext.scala:2164)

            at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1004)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)

            at org.apache.spark.rdd.RDD.withScope(RDD.scala:388)

            at org.apache.spark.rdd.RDD.collect(RDD.scala:1003)

            at org.apache.spark.sql.execution.SparkPlan.executeCollectIterator(SparkPlan.scala:392)

            at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.$anonfun$relationFuture$1(BroadcastExchangeExec.scala:120)

            at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:182)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Reply | Threaded
Open this post in threaded view
|

Re: Issue with executer

Sachit Murarka
Hi Vikas

1. Are you running in local mode? Master has local[*]
2. Pls mask the ip or confidential info while sharing logs

Thanks
Sachit

On Wed, 20 Jan 2021, 17:35 Vikas Garg, <[hidden email]> wrote:
Hi,

I am facing issue with spark executor. I am struggling with this issue since last many days and unable to resolve the issue.

Below is the configuration I have given.

  val spark = SparkSession.builder()
    .appName("Spark Job")
    .master("local[*]")
    .config("spark.dynamicAllocation.enabled", true)
    .config("spark.shuffle.service.enabled", true)
    .config("spark.driver.maxResultSize", "8g")
    .config("spark.driver.memory", "8g")
    .config("spark.executor.memory", "8g")
    .config("spark.network.timeout", "3600s")
    .getOrCreate()

1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

            at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

            at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

            at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

            at scala.Option.orElse(Option.scala:289)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

            at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

            at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

            at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

            at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

            at scala.Option.orElse(Option.scala:289)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

            at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketC

Reply | Threaded
Open this post in threaded view
|

Re: Issue with executer

sperry.it@gmail.com
In reply to this post by sperry.it@gmail.com
Hi Sean,

This works fine when I run with very less number of files. Looks like the issue is with memory or may be something else. I am unable to figure it out.

On Wed, 20 Jan 2021 at 17:58, Sean Owen <[hidden email]> wrote:
Does not look related to Spark - you have network connectivity problems.

On Wed, Jan 20, 2021 at 6:05 AM Vikas Garg <[hidden email]> wrote:
Hi,

I am facing issue with spark executor. I am struggling with this issue since last many days and unable to resolve the issue.

Below is the configuration I have given.

  val spark = SparkSession.builder()
    .appName("Spark Job")
    .master("local[*]")
    .config("spark.dynamicAllocation.enabled", true)
    .config("spark.shuffle.service.enabled", true)
    .config("spark.driver.maxResultSize", "8g")
    .config("spark.driver.memory", "8g")
    .config("spark.executor.memory", "8g")
    .config("spark.network.timeout", "3600s")
    .getOrCreate()

1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

            at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

            at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

            at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

            at scala.Option.orElse(Option.scala:289)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

            at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

            at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

            at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

            at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

            at scala.Option.orElse(Option.scala:289)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

            at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:12 ERROR TaskSetManager: Task 0 in stage 23.0 failed 1 times; aborting job

21/01/20 17:07:12 ERROR TaskSetManager: Task 2 in stage 17.0 failed 1 times; aborting job

Exception in thread "main" org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree:

Exchange hashpartitioning(claimid#1012, 200), true, [id=#742]

 

 

            at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56)

            at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.doExecute(ShuffleExchangeExec.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

            at org.apache.spark.sql.execution.InputAdapter.inputRDD(WholeStageCodegenExec.scala:525)

            at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(WholeStageCodegenExec.scala:453)

            at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(WholeStageCodegenExec.scala:452)

            at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:496)

            at org.apache.spark.sql.execution.SortExec.inputRDDs(SortExec.scala:132)

            at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:720)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

            at org.apache.spark.sql.execution.joins.SortMergeJoinExec.doExecute(SortMergeJoinExec.scala:182)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

            at org.apache.spark.sql.execution.ProjectExec.doExecute(basicPhysicalOperators.scala:76)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

            at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:316)

            at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:434)

            at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:420)

            at org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:47)

            at org.apache.spark.sql.Dataset.collectFromPlan(Dataset.scala:3627)

            at org.apache.spark.sql.Dataset.$anonfun$head$1(Dataset.scala:2697)

            at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3618)

            at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)

            at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)

            at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)

            at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)

            at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)

            at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3616)

            at org.apache.spark.sql.Dataset.head(Dataset.scala:2697)

            at org.apache.spark.sql.Dataset.take(Dataset.scala:2904)

            at org.apache.spark.sql.Dataset.getRows(Dataset.scala:300)

            at org.apache.spark.sql.Dataset.showString(Dataset.scala:337)

            at org.apache.spark.sql.Dataset.show(Dataset.scala:824)

            at org.apache.spark.sql.Dataset.show(Dataset.scala:783)

            at org.apache.spark.sql.Dataset.show(Dataset.scala:792)

            at com.brighthealthplan.cognizant.output.MedicalClaim$.getDS(MedicalClaim.scala:257)

            at com.brighthealthplan.cognizant.sparkjobs.DataManager.writeLeft(DataManager.scala:192)

            at com.brighthealthplan.cognizant.sparkjobs.Main$.main(Main.scala:107)

            at com.brighthealthplan.cognizant.sparkjobs.Main.main(Main.scala)

Caused by: java.util.concurrent.ExecutionException: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 23.0 failed 1 times, most recent failure: Lost task 0.0 in stage 23.0 (TID 29, del1-lhp-n99999.synapse.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

            at java.util.concurrent.FutureTask.report(FutureTask.java:122)

            at java.util.concurrent.FutureTask.get(FutureTask.java:206)

            at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(BroadcastExchangeExec.scala:195)

            at org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(WholeStageCodegenExec.scala:515)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:188)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:184)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(BroadcastHashJoinExec.scala:116)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(BroadcastHashJoinExec.scala:258)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:101)

            at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

            at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

            at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.consume(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(BroadcastHashJoinExec.scala:302)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:101)

            at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

            at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

            at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.consume(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(BroadcastHashJoinExec.scala:302)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(BroadcastHashJoinExec.scala:101)

            at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

            at org.apache.spark.sql.execution.ProjectExec.consume(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.ProjectExec.doConsume(basicPhysicalOperators.scala:71)

            at org.apache.spark.sql.execution.CodegenSupport.consume(WholeStageCodegenExec.scala:194)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(WholeStageCodegenExec.scala:149)

            at org.apache.spark.sql.execution.InputAdapter.consume(WholeStageCodegenExec.scala:496)

            at org.apache.spark.sql.execution.InputRDDCodegen.doProduce(WholeStageCodegenExec.scala:483)

            at org.apache.spark.sql.execution.InputRDDCodegen.doProduce$(WholeStageCodegenExec.scala:456)

            at org.apache.spark.sql.execution.InputAdapter.doProduce(WholeStageCodegenExec.scala:496)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.InputAdapter.produce(WholeStageCodegenExec.scala:496)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(BroadcastHashJoinExec.scala:95)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(BroadcastHashJoinExec.scala:39)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(basicPhysicalOperators.scala:51)

            at org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(WholeStageCodegenExec.scala:95)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.CodegenSupport.produce(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(WholeStageCodegenExec.scala:90)

            at org.apache.spark.sql.execution.ProjectExec.produce(basicPhysicalOperators.scala:41)

            at org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(WholeStageCodegenExec.scala:632)

            at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:692)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)

            at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)

            at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD$lzycompute(ShuffleExchangeExec.scala:106)

            at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD(ShuffleExchangeExec.scala:106)

            at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency$lzycompute(ShuffleExchangeExec.scala:139)

            at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency(ShuffleExchangeExec.scala:137)

            at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.$anonfun$doExecute$1(ShuffleExchangeExec.scala:154)

            at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52)

            ... 53 more

Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 23.0 failed 1 times, most recent failure: Lost task 0.0 in stage 23.0 (TID 29, del1-lhp-n99999.synapse.com, executor driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

            at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2059)

            at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2008)

            at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2007)

            at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:59)

            at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:52)

            at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)

            at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2007)

            at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:973)

            at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:973)

            at scala.Option.foreach(Option.scala:257)

            at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:973)

            at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2239)

            at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2188)

            at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2177)

            at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)

            at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:775)

            at org.apache.spark.SparkContext.runJob(SparkContext.scala:2099)

            at org.apache.spark.SparkContext.runJob(SparkContext.scala:2120)

            at org.apache.spark.SparkContext.runJob(SparkContext.scala:2139)

            at org.apache.spark.SparkContext.runJob(SparkContext.scala:2164)

            at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1004)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)

            at org.apache.spark.rdd.RDD.withScope(RDD.scala:388)

            at org.apache.spark.rdd.RDD.collect(RDD.scala:1003)

            at org.apache.spark.sql.execution.SparkPlan.executeCollectIterator(SparkPlan.scala:392)

            at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.$anonfun$relationFuture$1(BroadcastExchangeExec.scala:120)

            at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:182)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Reply | Threaded
Open this post in threaded view
|

Re: Issue with executer

sperry.it@gmail.com
In reply to this post by Sachit Murarka
Hi Sachit,

I am running it in local. My IP mentioned is the private IP address and therefore it is useless for anyone.

On Wed, 20 Jan 2021 at 17:37, Sachit Murarka <[hidden email]> wrote:
Hi Vikas

1. Are you running in local mode? Master has local[*]
2. Pls mask the ip or confidential info while sharing logs

Thanks
Sachit

On Wed, 20 Jan 2021, 17:35 Vikas Garg, <[hidden email]> wrote:
Hi,

I am facing issue with spark executor. I am struggling with this issue since last many days and unable to resolve the issue.

Below is the configuration I have given.

  val spark = SparkSession.builder()
    .appName("Spark Job")
    .master("local[*]")
    .config("spark.dynamicAllocation.enabled", true)
    .config("spark.shuffle.service.enabled", true)
    .config("spark.driver.maxResultSize", "8g")
    .config("spark.driver.memory", "8g")
    .config("spark.executor.memory", "8g")
    .config("spark.network.timeout", "3600s")
    .getOrCreate()

1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

            at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

            at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

            at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

            at scala.Option.orElse(Option.scala:289)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

            at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

            at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

            at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

            at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

            at scala.Option.orElse(Option.scala:289)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

            at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketC

Reply | Threaded
Open this post in threaded view
|

Re: Issue with executer

Mich Talebzadeh
Hi Vikas,

Are you running this on your local laptop etc or using some IDE etc?

What is your available memory for Spark?

Start with minimum set like below

def spark_session_local(appName):
    return SparkSession.builder \
        .master('local[1]') \
        .appName(appName) \
        .enableHiveSupport() \
        .getOrCreate()


HTH



LinkedIn  https://www.linkedin.com/profile/view?id=AAEAAAAWh2gBxianrbJd6zP6AcPCCdOABUrV8Pw

 



Disclaimer: Use it at your own risk. Any and all responsibility for any loss, damage or destruction of data or any other property which may arise from relying on this email's technical content is explicitly disclaimed. The author will in no case be liable for any monetary damages arising from such loss, damage or destruction.

 



On Wed, 20 Jan 2021 at 12:32, Vikas Garg <[hidden email]> wrote:
Hi Sachit,

I am running it in local. My IP mentioned is the private IP address and therefore it is useless for anyone.

On Wed, 20 Jan 2021 at 17:37, Sachit Murarka <[hidden email]> wrote:
Hi Vikas

1. Are you running in local mode? Master has local[*]
2. Pls mask the ip or confidential info while sharing logs

Thanks
Sachit

On Wed, 20 Jan 2021, 17:35 Vikas Garg, <[hidden email]> wrote:
Hi,

I am facing issue with spark executor. I am struggling with this issue since last many days and unable to resolve the issue.

Below is the configuration I have given.

  val spark = SparkSession.builder()
    .appName("Spark Job")
    .master("local[*]")
    .config("spark.dynamicAllocation.enabled", true)
    .config("spark.shuffle.service.enabled", true)
    .config("spark.driver.maxResultSize", "8g")
    .config("spark.driver.memory", "8g")
    .config("spark.executor.memory", "8g")
    .config("spark.network.timeout", "3600s")
    .getOrCreate()

1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

            at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

            at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

            at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

            at scala.Option.orElse(Option.scala:289)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

            at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

            at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

            at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

            at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

            at scala.Option.orElse(Option.scala:289)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

            at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketC

Reply | Threaded
Open this post in threaded view
|

Re: Issue with executer

sperry.it@gmail.com
The issue is resolved. Resolution is little weird but it worked. It was due to scala version mismatch with projects in my package.

On Wed, 20 Jan 2021 at 18:07, Mich Talebzadeh <[hidden email]> wrote:
Hi Vikas,

Are you running this on your local laptop etc or using some IDE etc?

What is your available memory for Spark?

Start with minimum set like below

def spark_session_local(appName):
    return SparkSession.builder \
        .master('local[1]') \
        .appName(appName) \
        .enableHiveSupport() \
        .getOrCreate()


HTH



LinkedIn  https://www.linkedin.com/profile/view?id=AAEAAAAWh2gBxianrbJd6zP6AcPCCdOABUrV8Pw

 



Disclaimer: Use it at your own risk. Any and all responsibility for any loss, damage or destruction of data or any other property which may arise from relying on this email's technical content is explicitly disclaimed. The author will in no case be liable for any monetary damages arising from such loss, damage or destruction.

 



On Wed, 20 Jan 2021 at 12:32, Vikas Garg <[hidden email]> wrote:
Hi Sachit,

I am running it in local. My IP mentioned is the private IP address and therefore it is useless for anyone.

On Wed, 20 Jan 2021 at 17:37, Sachit Murarka <[hidden email]> wrote:
Hi Vikas

1. Are you running in local mode? Master has local[*]
2. Pls mask the ip or confidential info while sharing logs

Thanks
Sachit

On Wed, 20 Jan 2021, 17:35 Vikas Garg, <[hidden email]> wrote:
Hi,

I am facing issue with spark executor. I am struggling with this issue since last many days and unable to resolve the issue.

Below is the configuration I have given.

  val spark = SparkSession.builder()
    .appName("Spark Job")
    .master("local[*]")
    .config("spark.dynamicAllocation.enabled", true)
    .config("spark.shuffle.service.enabled", true)
    .config("spark.driver.maxResultSize", "8g")
    .config("spark.driver.memory", "8g")
    .config("spark.executor.memory", "8g")
    .config("spark.network.timeout", "3600s")
    .getOrCreate()

1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

            at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

            at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

            at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

            at scala.Option.orElse(Option.scala:289)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

            at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(RetryingBlockFetcher.java:121)

            at org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(NettyBlockTransferService.scala:143)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(BlockTransferService.scala:103)

            at org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(BlockManager.scala:1010)

            at org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(BlockManager.scala:954)

            at scala.Option.orElse(Option.scala:289)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(BlockManager.scala:954)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(BlockManager.scala:1092)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:88)

            at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 1 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 2 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330)

            at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)

            at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)

            at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning fetch of 1 outstanding blocks (after 3 retries)

java.io.IOException: Failed to connect to del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:253)

            at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:195)

            at org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(NettyBlockTransferService.scala:122)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:141)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:169)

            at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

            at java.util.concurrent.FutureTask.run(FutureTask.java:266)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)

            at java.lang.Thread.run(Thread.java:748)

Caused by: io.netty.channel.AbstractChannel$AnnotatedSocketException: Permission denied: no further information: del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: java.net.SocketException: Permission denied: no further information

            at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)

            at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:715)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketC