如何通过livy Programmatic API提交批处理jar Spark作业

您可以使用Livy通过rest API提交spark作业。请按照以下步骤操作，

首先构建spark应用程序并创建程序集jar并将应用程序jar上载到hadoop集群的集群存储（HDFS）上。
使用curl（用于测试）提交作业并使用http client api实现。
使用scala中的http客户端提交spark作业的示例代码

import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet,
HttpPost, HttpPut}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.{CloseableHttpClient, HttpClientBuilder}
import org.apache.http.util.EntityUtils

import scala.util.parsing.json.{JSON, JSONObject}

def submitJob(className: String, jarPath:String, extraArgs: List[String]) : JSONObject = {

val jobSubmitRequest = new HttpPost(s"${clusterConfig.livyserver}/batches")

val data = Map(
"className"-> className,
"file" -> jarPath,
"driverMemory" -> "2g",
"name" -> "LivyTest",
"proxyUser" -> "hadoop")

if(extraArgs != null && !extraArgs.isEmpty) {
data + ( "args" -> extraArgs)
}

val json = new JSONObject(data)

println(json.toString())

val params = new StringEntity(json.toString(),"UTF-8")
params.setContentType("application/json")

jobSubmitRequest.addHeader("Content-Type", "application/json")
jobSubmitRequest.addHeader("Accept", "/")
jobSubmitRequest.setEntity(params)

val client: CloseableHttpClient = HttpClientBuilder.create().build()
val response: CloseableHttpResponse = client.execute(jobSubmitRequest)
HttpReqUtil.parseHttpResponse(response)._2
}