Spark Streaming with Confluent

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
2 messages Options
Reply | Threaded
Open this post in threaded view
|

Spark Streaming with Confluent

Arkadiusz Bicz
Hi,

I try to test spark streaming 2.2.0 version with confluent 3.3.0 

I have got lot of error during compilation this is my sbt:

lazy val sparkstreaming = (project in file("."))
  .settings(
      name := "sparkstreaming",
  organization := "org.arek",
  version := "0.1-SNAPSHOT",
  scalaVersion := "2.11.8",
    libraryDependencies ++=  Seq(
      "org.apache.spark" %% "spark-streaming" % "2.2.0",
      "org.apache.spark" %% "spark-streaming-kafka-0-8" % "2.2.0",
      "io.confluent" % "kafka-avro-serializer" % "3.3.0"
    )
  )


import org.apache.spark._
import org.apache.spark.streaming._
import org.apache.spark.streaming.kafka.KafkaUtils
import io.confluent.kafka.serializers.KafkaAvroDecoder

object Transformation extends Serializable {

  def main(args: Array[String]) = {
    val conf = new SparkConf().setAppName("StreamingTranformation").setMaster("local[*]")
    val streamingContext = new StreamingContext(conf, Seconds(1))

    val kafkaParams = Map[String, String]("metadata.broker.list" -> "local:2181",
      "schema.registry.url" -> "http://local:8081",
      "auto.offset.reset" -> "smallest")

    val topicSet = Set("GEXPPROD_ROUTE")
    val messages = KafkaUtils.createDirectStream[Object, Object, KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams, topicSet).map(_._2)

    val lines = messages.foreachRDD(rdd => {
      rdd.foreach({ avroRecord =>
        println(avroRecord)
      })
    })
  }


[warn] Found version conflict(s) in library dependencies; some are suspected to be binary incompatible:
[warn]  * io.netty:netty:3.9.9.Final is selected over {3.6.2.Final, 3.7.0.Final}
[warn]      +- org.apache.spark:spark-core_2.11:2.2.0             (depends on 3.7.0.Final)
[warn]      +- org.apache.zookeeper:zookeeper:3.4.8               (depends on 3.7.0.Final)
[warn]      +- org.apache.zookeeper:zookeeper:3.4.6               (depends on 3.6.2.Final)
[warn]      +- org.apache.hadoop:hadoop-hdfs:2.6.5                (depends on 3.6.2.Final)
[warn]  * commons-net:commons-net:2.2 is selected over 3.1
[warn]      +- org.apache.spark:spark-core_2.11:2.2.0             (depends on 3.1)
[warn]      +- org.apache.hadoop:hadoop-common:2.6.5              (depends on 3.1)
[warn]  * com.google.guava:guava:11.0.2 is selected over {12.0.1, 16.0.1}
[warn]      +- org.apache.hadoop:hadoop-yarn-client:2.6.5         (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-api:2.6.5            (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-common:2.6.5         (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-server-nodemanager:2.6.5 (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-common:2.6.5              (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-server-common:2.6.5  (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-hdfs:2.6.5                (depends on 11.0.2)
[warn]      +- org.apache.curator:curator-framework:2.6.0         (depends on 16.0.1)
[warn]      +- org.apache.curator:curator-client:2.6.0            (depends on 16.0.1)
[warn]      +- org.apache.curator:curator-recipes:2.6.0           (depends on 16.0.1)
[warn]      +- org.htrace:htrace-core:3.0.4                       (depends on 12.0.1)
[warn] Run 'evicted' to see detailed eviction warnings
[info] Compiling 1 Scala source to /home/adminuser/data-streaming-platform/sparkstreaming/target/scala-2.11/classes ...
[error] /home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:6:11: object confluent is not a member of package org.apache.spark.io
[error] import io.confluent.kafka.serializers.KafkaAvroDecoder
[error]           ^
[error] /home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:19:66: not found: type KafkaAvroDecoder
[error]     val messages = KafkaUtils.createDirectStream[Object, Object, KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams, topicSet).map(_._2)
[error]                                                                  ^
[error] /home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:19:84: not found: type KafkaAvroDecoder
[error]     val messages = KafkaUtils.createDirectStream[Object, Object, KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams, topicSet).map(_._2)
[error]               


When changing to library  "org.apache.spark" %% "spark-streaming-kafka-0-10" % "2.2.0" :


[warn] Found version conflict(s) in library dependencies; some are suspected to be binary incompatible:
[warn]  * com.101tec:zkclient:0.10 is selected over 0.8
[warn]      +- io.confluent:common-utils:3.3.0                    (depends on 0.10)
[warn]      +- org.apache.kafka:kafka_2.11:0.10.0.1               (depends on 0.8)
[warn]  * io.netty:netty:3.9.9.Final is selected over {3.6.2.Final, 3.7.0.Final}
[warn]      +- org.apache.spark:spark-core_2.11:2.2.0             (depends on 3.7.0.Final)
[warn]      +- org.apache.zookeeper:zookeeper:3.4.8               (depends on 3.7.0.Final)
[warn]      +- org.apache.zookeeper:zookeeper:3.4.6               (depends on 3.6.2.Final)
[warn]      +- org.apache.hadoop:hadoop-hdfs:2.6.5                (depends on 3.6.2.Final)
[warn]  * commons-net:commons-net:2.2 is selected over 3.1
[warn]      +- org.apache.spark:spark-core_2.11:2.2.0             (depends on 3.1)
[warn]      +- org.apache.hadoop:hadoop-common:2.6.5              (depends on 3.1)
[warn]  * com.google.guava:guava:11.0.2 is selected over {12.0.1, 16.0.1}
[warn]      +- org.apache.hadoop:hadoop-yarn-client:2.6.5         (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-api:2.6.5            (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-common:2.6.5         (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-server-nodemanager:2.6.5 (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-common:2.6.5              (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-server-common:2.6.5  (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-hdfs:2.6.5                (depends on 11.0.2)
[warn]      +- org.apache.curator:curator-framework:2.6.0         (depends on 16.0.1)
[warn]      +- org.apache.curator:curator-client:2.6.0            (depends on 16.0.1)
[warn]      +- org.apache.curator:curator-recipes:2.6.0           (depends on 16.0.1)
[warn]      +- org.htrace:htrace-core:3.0.4                       (depends on 12.0.1)
[warn] Run 'evicted' to see detailed eviction warnings
[info] Compiling 1 Scala source to /home/adminuser/data-streaming-platform/sparkstreaming/target/scala-2.11/classes ...
[error] /home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:18:11: object confluent is not a member of package org.apache.spark.io
[error] import io.confluent.kafka.serializers.KafkaAvroDecoder

Regards,

Arek 
                                                                     ^
Reply | Threaded
Open this post in threaded view
|

Re: Spark Streaming with Confluent

maasg
Hi Arkadiusz, 

Try 'rooting' your import. It looks like the import is being interpreted as being relative to the previous. 
'rooting; is done by adding the  '_root_'  prefix to your import:

import org.apache.spark.streaming.kafka.KafkaUtils
import _root_.io.confluent.kafka.serializers.KafkaAvroDecoder

kr, Gerard.

On Wed, Dec 13, 2017 at 6:05 PM, Arkadiusz Bicz <[hidden email]> wrote:
Hi,

I try to test spark streaming 2.2.0 version with confluent 3.3.0 

I have got lot of error during compilation this is my sbt:

lazy val sparkstreaming = (project in file("."))
  .settings(
      name := "sparkstreaming",
  organization := "org.arek",
  version := "0.1-SNAPSHOT",
  scalaVersion := "2.11.8",
    libraryDependencies ++=  Seq(
      "org.apache.spark" %% "spark-streaming" % "2.2.0",
      "org.apache.spark" %% "spark-streaming-kafka-0-8" % "2.2.0",
      "io.confluent" % "kafka-avro-serializer" % "3.3.0"
    )
  )


import org.apache.spark._
import org.apache.spark.streaming._
import org.apache.spark.streaming.kafka.KafkaUtils
import io.confluent.kafka.serializers.KafkaAvroDecoder

object Transformation extends Serializable {

  def main(args: Array[String]) = {
    val conf = new SparkConf().setAppName("StreamingTranformation").setMaster("local[*]")
    val streamingContext = new StreamingContext(conf, Seconds(1))

    val kafkaParams = Map[String, String]("metadata.broker.list" -> "local:2181",
      "schema.registry.url" -> "http://local:8081",
      "auto.offset.reset" -> "smallest")

    val topicSet = Set("GEXPPROD_ROUTE")
    val messages = KafkaUtils.createDirectStream[Object, Object, KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams, topicSet).map(_._2)

    val lines = messages.foreachRDD(rdd => {
      rdd.foreach({ avroRecord =>
        println(avroRecord)
      })
    })
  }


[warn] Found version conflict(s) in library dependencies; some are suspected to be binary incompatible:
[warn]  * io.netty:netty:3.9.9.Final is selected over {3.6.2.Final, 3.7.0.Final}
[warn]      +- org.apache.spark:spark-core_2.11:2.2.0             (depends on 3.7.0.Final)
[warn]      +- org.apache.zookeeper:zookeeper:3.4.8               (depends on 3.7.0.Final)
[warn]      +- org.apache.zookeeper:zookeeper:3.4.6               (depends on 3.6.2.Final)
[warn]      +- org.apache.hadoop:hadoop-hdfs:2.6.5                (depends on 3.6.2.Final)
[warn]  * commons-net:commons-net:2.2 is selected over 3.1
[warn]      +- org.apache.spark:spark-core_2.11:2.2.0             (depends on 3.1)
[warn]      +- org.apache.hadoop:hadoop-common:2.6.5              (depends on 3.1)
[warn]  * com.google.guava:guava:11.0.2 is selected over {12.0.1, 16.0.1}
[warn]      +- org.apache.hadoop:hadoop-yarn-client:2.6.5         (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-api:2.6.5            (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-common:2.6.5         (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-server-nodemanager:2.6.5 (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-common:2.6.5              (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-server-common:2.6.5  (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-hdfs:2.6.5                (depends on 11.0.2)
[warn]      +- org.apache.curator:curator-framework:2.6.0         (depends on 16.0.1)
[warn]      +- org.apache.curator:curator-client:2.6.0            (depends on 16.0.1)
[warn]      +- org.apache.curator:curator-recipes:2.6.0           (depends on 16.0.1)
[warn]      +- org.htrace:htrace-core:3.0.4                       (depends on 12.0.1)
[warn] Run 'evicted' to see detailed eviction warnings
[info] Compiling 1 Scala source to /home/adminuser/data-streaming-platform/sparkstreaming/target/scala-2.11/classes ...
[error] /home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:6:11: object confluent is not a member of package org.apache.spark.io
[error] import io.confluent.kafka.serializers.KafkaAvroDecoder
[error]           ^
[error] /home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:19:66: not found: type KafkaAvroDecoder
[error]     val messages = KafkaUtils.createDirectStream[Object, Object, KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams, topicSet).map(_._2)
[error]                                                                  ^
[error] /home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:19:84: not found: type KafkaAvroDecoder
[error]     val messages = KafkaUtils.createDirectStream[Object, Object, KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams, topicSet).map(_._2)
[error]               


When changing to library  "org.apache.spark" %% "spark-streaming-kafka-0-10" % "2.2.0" :


[warn] Found version conflict(s) in library dependencies; some are suspected to be binary incompatible:
[warn]  * com.101tec:zkclient:0.10 is selected over 0.8
[warn]      +- io.confluent:common-utils:3.3.0                    (depends on 0.10)
[warn]      +- org.apache.kafka:kafka_2.11:0.10.0.1               (depends on 0.8)
[warn]  * io.netty:netty:3.9.9.Final is selected over {3.6.2.Final, 3.7.0.Final}
[warn]      +- org.apache.spark:spark-core_2.11:2.2.0             (depends on 3.7.0.Final)
[warn]      +- org.apache.zookeeper:zookeeper:3.4.8               (depends on 3.7.0.Final)
[warn]      +- org.apache.zookeeper:zookeeper:3.4.6               (depends on 3.6.2.Final)
[warn]      +- org.apache.hadoop:hadoop-hdfs:2.6.5                (depends on 3.6.2.Final)
[warn]  * commons-net:commons-net:2.2 is selected over 3.1
[warn]      +- org.apache.spark:spark-core_2.11:2.2.0             (depends on 3.1)
[warn]      +- org.apache.hadoop:hadoop-common:2.6.5              (depends on 3.1)
[warn]  * com.google.guava:guava:11.0.2 is selected over {12.0.1, 16.0.1}
[warn]      +- org.apache.hadoop:hadoop-yarn-client:2.6.5         (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-api:2.6.5            (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-common:2.6.5         (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-server-nodemanager:2.6.5 (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-common:2.6.5              (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-yarn-server-common:2.6.5  (depends on 11.0.2)
[warn]      +- org.apache.hadoop:hadoop-hdfs:2.6.5                (depends on 11.0.2)
[warn]      +- org.apache.curator:curator-framework:2.6.0         (depends on 16.0.1)
[warn]      +- org.apache.curator:curator-client:2.6.0            (depends on 16.0.1)
[warn]      +- org.apache.curator:curator-recipes:2.6.0           (depends on 16.0.1)
[warn]      +- org.htrace:htrace-core:3.0.4                       (depends on 12.0.1)
[warn] Run 'evicted' to see detailed eviction warnings
[info] Compiling 1 Scala source to /home/adminuser/data-streaming-platform/sparkstreaming/target/scala-2.11/classes ...
[error] /home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:18:11: object confluent is not a member of package org.apache.spark.io
[error] import io.confluent.kafka.serializers.KafkaAvroDecoder

Regards,

Arek 
                                                                     ^