HiveContext

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

def analyze(tableName: String): Unit

Analyzes the given table in the current database to generate statistics, which will be used in query optimizations.

lazy val analyzer: Analyzer

def applySchemaToPythonRDD(rdd: RDD[Array[Any]], schema: StructType): DataFrame

Apply a schema defined by the schema to an RDD.

def applySchemaToPythonRDD(rdd: RDD[Array[Any]], schemaString: String): DataFrame

Apply a schema defined by the schemaString to an RDD.

final def asInstanceOf[T0]: T0

def baseRelationToDataFrame(baseRelation: BaseRelation): DataFrame

Convert a BaseRelation created for external data sources into a DataFrame.

val cacheManager: execution.CacheManager

def cacheTable(tableName: String): Unit

Caches the specified table in-memory.

lazy val catalog: HiveMetastoreCatalog with OverrideCatalog

def clearCache(): Unit

Removes all cached tables from the in-memory cache.

def clone(): AnyRef

def conf: SQLConf

def configure(): Map[String, String]

Overridden by child classes that need to set configuration before the client init.

def convertCTAS: Boolean

When true, a table created by a Hive CTAS statement (no USING clause) will be converted to a data source table, using the data source set by spark.

def convertMetastoreParquet: Boolean

When true, enables an experimental feature where metastore tables that use the parquet SerDe are automatically converted to use the Spark SQL parquet table scan, instead of the Hive SerDe.

def convertMetastoreParquetWithSchemaMerging: Boolean

When true, also tries to merge possibly different but compatible Parquet schemas in different Parquet data files.

def createDataFrame(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

Applies a schema to an RDD of Java Beans.

def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame

Applies a schema to an RDD of Java Beans.

def createDataFrame(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

:: DeveloperApi :: Creates a DataFrame from an JavaRDD containing Rows using the given schema.

def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame

:: DeveloperApi :: Creates a DataFrame from an RDD containing Rows using the given schema.

import org.apache.spark.sql._
import org.apache.spark.sql.types._
val sqlContext = new org.apache.spark.sql.SQLContext(sc)

val schema =
  StructType(
    StructField("name", StringType, false) ::
    StructField("age", IntegerType, true) :: Nil)

val people =
  sc.textFile("examples/src/main/resources/people.txt").map(
    _.split(",")).map(p => Row(p(0), p(1).trim.toInt))
val dataFrame = sqlContext.createDataFrame(people, schema)
dataFrame.printSchema
// root
// |-- name: string (nullable = false)
// |-- age: integer (nullable = true)

dataFrame.registerTempTable("people")
sqlContext.sql("select name from people").collect.foreach(println)

def createDataFrame[A <: Product](data: Seq[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

:: Experimental :: Creates a DataFrame from a local Seq of Product.

def createDataFrame[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

:: Experimental :: Creates a DataFrame from an RDD of case classes.

def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame

:: Experimental :: (Scala-specific) Create an external table from the given path based on a data source, a schema and a set of options.

def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame

:: Experimental :: Create an external table from the given path based on a data source, a schema and a set of options.

def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame

:: Experimental :: (Scala-specific) Creates an external table from the given path based on a data source and a set of options.

def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame

:: Experimental :: Creates an external table from the given path based on a data source and a set of options.

def createExternalTable(tableName: String, path: String, source: String): DataFrame

:: Experimental :: Creates an external table from the given path based on a data source and returns the corresponding DataFrame.

def createExternalTable(tableName: String, path: String): DataFrame

:: Experimental :: Creates an external table from the given path and returns the corresponding DataFrame.

def createSession(): SQLSession

def currentSession(): HiveContext.SQLSession

val ddlParser: DDLParser

val defaultSession: HiveContext.SQLSession

def detachSession(): Unit

def dialectClassName: String

def dropTempTable(tableName: String): Unit

Drops the temporary table with the given table name in the catalog.

lazy val emptyDataFrame: DataFrame

:: Experimental :: Returns a DataFrame with no rows or columns.

lazy val emptyResult: RDD[Row]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def executePlan(plan: LogicalPlan): QueryExecution

def executeSql(sql: String): HiveContext.QueryExecution

lazy val executionHive: ClientWrapper

The copy of the hive client that is used for execution.

val experimental: ExperimentalMethods

:: Experimental :: A collection of methods that are considered experimental, but can be used to hook into the query planner for advanced functionality.

def finalize(): Unit

lazy val functionRegistry: HiveFunctionRegistry with OverrideFunctionRegistry

def getAllConfs: Map[String, String]

Return all the configuration properties that have been set (i.

final def getClass(): Class[_]

def getConf(key: String, defaultValue: String): String

Return the value of Spark SQL configuration property for the given key.

def getConf(key: String): String

Return the value of Spark SQL configuration property for the given key.

def getSQLDialect(): ParserDialect

def getSchema(beanClass: Class[_]): Seq[AttributeReference]

Returns a Catalyst Schema for the given java bean class.

def hashCode(): Int

def hiveMetastoreBarrierPrefixes: Seq[String]

A comma separated list of class prefixes that should explicitly be reloaded for each version of Hive that Spark SQL is communicating with.

def hiveMetastoreJars: String

The location of the jars that should be used to instantiate the HiveMetastoreClient.

def hiveMetastoreSharedPrefixes: Seq[String]

A comma separated list of class prefixes that should be loaded using the classloader that is shared between Spark SQL and a specific version of Hive.

def hiveMetastoreVersion: String

The version of the hive client that will be used to communicate with the metastore.

def hiveconf: HiveConf

object implicits extends Serializable

:: Experimental :: (Scala-specific) Implicit methods available in Scala for converting common Scala objects into DataFrames.

def invalidateTable(tableName: String): Unit

def isCached(tableName: String): Boolean

Returns true if the table is currently cached in-memory.

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

lazy val metadataHive: ClientInterface

The copy of the Hive client that is used to retrieve metadata from the Hive MetaStore.

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def openSession(): HiveContext.SQLSession

lazy val optimizer: Optimizer

def parseDataType(dataTypeString: String): DataType

Parses the data type in our internal string representation.

def parseSql(sql: String): LogicalPlan

val planner: SparkPlanner with HiveStrategies

val prepareForExecution: RuleExecutor[SparkPlan] { val batches: List[this.Batch] }

Prepares a planned SparkPlan for execution by inserting shuffle operations as needed.

def range(start: Long, end: Long, step: Long, numPartitions: Int): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in an range from start to end(exclusive) with an step value, with partition number specified.

def range(end: Long): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in an range from 0 to end(exclusive) with step value 1.

def range(start: Long, end: Long): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in an range from start to end(exclusive) with step value 1.

def read: DataFrameReader

:: Experimental :: Returns a DataFrameReader that can be used to read data in as a DataFrame.

sqlContext.read.parquet("/path/to/file.parquet")
sqlContext.read.schema(schema).json("/path/to/file.json")

def refreshTable(tableName: String): Unit

Invalidate and refresh all the cached the metadata of the given table.

def runSqlHive(sql: String): Seq[String]

def setConf(key: String, value: String): Unit

Set the given Spark SQL configuration property.

def setConf(props: Properties): Unit

Set Spark SQL configuration properties.

val sparkContext: SparkContext

def sql(sqlText: String): DataFrame

Executes a SQL query using Spark, returning the result as a DataFrame.

val sqlParser: SparkSQLParser

lazy val substitutor: VariableSubstitution

final def synchronized[T0](arg0: ⇒ T0): T0

def table(tableName: String): DataFrame

Returns the specified table as a DataFrame.

def tableNames(databaseName: String): Array[String]

Returns the names of tables in the given database as an array.

def tableNames(): Array[String]

Returns the names of tables in the current database as an array.

def tables(databaseName: String): DataFrame

Returns a DataFrame containing names of existing tables in the given database.

def tables(): DataFrame

Returns a DataFrame containing names of existing tables in the current database.

val tlSession: ThreadLocal[HiveContext.SQLSession]

def toString(): String

val udf: UDFRegistration

A collection of methods for registering user-defined functions (UDF).

sqlContext.udf.register("myUdf", (arg1: Int, arg2: String) => arg2 + arg1)

sqlContext.udf().register("myUDF",
 new UDF2<Integer, String, String>() {
     @Override
     public String call(Integer arg1, String arg2) {
         return arg2 + arg1;
     }
}, DataTypes.StringType);

sqlContext.udf().register("myUDF",
(Integer arg1, String arg2) -> arg2 + arg1),
DataTypes.StringType);

def uncacheTable(tableName: String): Unit

Removes the specified table from the in-memory cache.

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def applySchema(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

def applySchema(rdd: RDD[_], beanClass: Class[_]): DataFrame

def applySchema(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

def applySchema(rowRDD: RDD[Row], schema: StructType): DataFrame

def jdbc(url: String, table: String, theParts: Array[String]): DataFrame

Construct a DataFrame representing the database table accessible via JDBC URL url named table.

def jdbc(url: String, table: String, columnName: String, lowerBound: Long, upperBound: Long, numPartitions: Int): DataFrame

Construct a DataFrame representing the database table accessible via JDBC URL url named table.

def jdbc(url: String, table: String): DataFrame

Construct a DataFrame representing the database table accessible via JDBC URL url named table.

def jsonFile(path: String, samplingRatio: Double): DataFrame

def jsonFile(path: String, schema: StructType): DataFrame

Loads a JSON file (one object per line) and applies the given schema, returning the result as a DataFrame.

def jsonFile(path: String): DataFrame

Loads a JSON file (one object per line), returning the result as a DataFrame.

def jsonRDD(json: JavaRDD[String], samplingRatio: Double): DataFrame

Loads a JavaRDD[String] storing JSON objects (one object per record) inferring the schema, returning the result as a DataFrame.

def jsonRDD(json: RDD[String], samplingRatio: Double): DataFrame

Loads an RDD[String] storing JSON objects (one object per record) inferring the schema, returning the result as a DataFrame.

def jsonRDD(json: JavaRDD[String], schema: StructType): DataFrame

Loads an JavaRDD<String> storing JSON objects (one object per record) and applies the given schema, returning the result as a DataFrame.

def jsonRDD(json: RDD[String], schema: StructType): DataFrame

Loads an RDD[String] storing JSON objects (one object per record) and applies the given schema, returning the result as a DataFrame.

def jsonRDD(json: JavaRDD[String]): DataFrame

Loads an RDD[String] storing JSON objects (one object per record), returning the result as a DataFrame.

def jsonRDD(json: RDD[String]): DataFrame

Loads an RDD[String] storing JSON objects (one object per record), returning the result as a DataFrame.

def load(source: String, schema: StructType, options: Map[String, String]): DataFrame

(Scala-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame, using the given schema as the schema of the DataFrame.

def load(source: String, schema: StructType, options: Map[String, String]): DataFrame

(Java-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame, using the given schema as the schema of the DataFrame.

def load(source: String, options: Map[String, String]): DataFrame

(Scala-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame.

def load(source: String, options: Map[String, String]): DataFrame

(Java-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame.

def load(path: String, source: String): DataFrame

Returns the dataset stored at path as a DataFrame, using the given data source.

def load(path: String): DataFrame

Returns the dataset stored at path as a DataFrame, using the default data source configured by spark.

def parquetFile(paths: String*): DataFrame

Loads a Parquet file, returning the result as a DataFrame.