'NoHostAvailableException spark-cassandra-connector
I am using sparkContext to count the rows of a cassandra table but getting below error :
com.datastax.driver.core.exceptions.NoHostAvailableException: All host(s) tried for query failed (no host was tried)
I am getting above error after CassandraConnector: Connected to Cassandra cluster: cluster_name .
Below is the configuration code . I read that I need to setup driver protocol but there is no option for it to add spark-cassandra-connector .
val sparkConf = new SparkConf(true).setAppName(appName)
sparkConf.set("spark.sql.orc.enabled", "true")
sparkConf.set("spark.sql.hive.convertMetastoreOrc", "false")
sparkConf.set("spark.sql.hive.metastorePartitionPruning true", "true")
sparkConf.set("spark.sql.orc.filterPushdown", "true")
sparkConf.set("spark.cassandra.connection.host", connection.hosts)
sparkConf.set("spark.cassandra.auth.username", connection.user)
sparkConf.set("spark.cassandra.auth.password", connection.password)
sparkConf.set("spark.cassandra.connection.local_dc", connection.local_dc.getOrElse(""))
sparkConf.set("spark.cassandra.input.consistency.level", connection.consistency_level)
sparkConf.set("spark.cassandra.input.reads_per_sec", Parameters.getCassandraInputReadsPerSec)
sparkConf.set("spark.cassandra.input.fetch.size_in_rows", Parameters.getCassandraInputFetchSizeInRows)
sparkConf.set("spark.cassandra.connection.timeout_ms", Parameters.getCassandraConnectionTimeoutMs)
sparkConf.set("spark.cassandra.read.timeout_ms", Parameters.getCassandraReadTimeoutMs)
sparkConf.set("spark.cassandra.connection.keep_alive_ms", Parameters.getCassandraConnectionKeepAliveMs)
val sparkSession: SparkSession = SparkSession.builder().enableHiveSupport().config(sparkConf).getOrCreate()
val sc: SparkContext = sparkSession.sparkContext
val sqlContext = sparkSession.sqlContext
sqlContext.setConf("fs.maprfs.impl", Parameters.getFsMaprfsImpl)
sqlContext.setConf("hive.exec.compress.output", Parameters.getHiveExecCompressOutput)
sqlContext.setConf("hive.merge.mapredfiles", Parameters.getHiveMergeMapredfiles)
sqlContext.setConf("hive.merge.smallfiles.avgsize", Parameters.getHiveMergeSmallfilesAvgsize)
sqlContext.setConf("hive.exec.reducers.bytes.per.reducer", Parameters.getHiveExecReducersBytesPerReducer)
sqlContext.setConf("hive.exec.dynamic.partition", Parameters.getHiveExecDynamicPartition)
sqlContext.setConf("hive.exec.dynamic.partition.mode", Parameters.getHiveExecDynamicPartitionMode)
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|
