本文主要是介绍Java调用oozie提交spark on yarn任务，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！
1.需要在oozie-site.xml设置如下属性：
<property> <name>hadoop.proxyuser.cenyuhai.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.cenyuhai.groups</name> <value>*</value> </property>
如果Oozie报错ClassNotFoundException: Class org.apache.oozie.action.hadoop.SparkMain 再添加一个属性
<property> <name>oozie.use.system.libpath</name> <value>true</value> </property>
3.Java调用代码
maven的jar吧
<dependency><groupId>org.apache.oozie</groupId><artifactId>oozie-client</artifactId><version>4.1.0-cdh5.9.0</version></dependency><dependency><groupId>org.apache.oozie</groupId><artifactId>oozie-core</artifactId><version>4.1.0-cdh5.9.0</version><scope>provided</scope></dependency><dependency><groupId>org.apache.oozie</groupId><artifactId>oozie-hadoop</artifactId><version>2.6.0-cdh5.9.2.oozie-4.1.0-cdh5.9.2</version><scope>provided</scope></dependency>import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.OozieClientException;
import org.apache.oozie.client.WorkflowJob;
import org.apache.oozie.client.WorkflowJob.Status;import java.util.LinkedList;
import java.util.List;
import java.util.Properties;/*** Created by root on 7/19/17.*/
public class WorkFlowClient {private static String OOZIE_URL = "http://master01:11000/oozie/";private static String JOB_PATH = "hdfs://master01:8020/bobo/in/flow/workflow.xml";private static String JOB_Tracker = "master01:8032";private static String NAMENode = "hdfs://master01:8020";OozieClient wc = null;public WorkFlowClient(String url){wc = new OozieClient(url);}public String startJob(String wfDefinition, List<WorkflowParameter> wfParameters)throws OozieClientException {// create a workflow job configuration and set the workflow application pathProperties conf = wc.createConfiguration();conf.setProperty(OozieClient.APP_PATH, wfDefinition);// setting workflow parametersconf.setProperty("jobTracker", JOB_Tracker);conf.setProperty("nameNode", NAMENode);if((wfParameters != null) && (wfParameters.size() > 0)){for(WorkflowParameter parameter : wfParameters)conf.setProperty(parameter.getName(), parameter.getValue());}// submit and start the workflow jobreturn wc.run(conf);}public Status getJobStatus(String jobID) throws OozieClientException{WorkflowJob job = wc.getJobInfo(jobID);return job.getStatus();}public static void main(String[] args) throws OozieClientException, InterruptedException{// Create clientWorkFlowClient client = new WorkFlowClient(OOZIE_URL);// Create parametersList<WorkflowParameter> wfParameters = new LinkedList<WorkflowParameter>();WorkflowParameter jobmaster = new WorkflowParameter("jobmaster","yarn-client");WorkflowParameter jobmode = new WorkflowParameter("jobmode","client");WorkflowParameter jobname = new WorkflowParameter("jobname","SparkOozieAction");WorkflowParameter jarclass = new WorkflowParameter("jarclass","com.kafkaspark.sparkstreaming.SparkstreamingData");WorkflowParameter jarpath = new WorkflowParameter("jarpath","hdfs://master01:8020/bobo/in/flow/kafkaspark.jar");WorkflowParameter sparkopts = new WorkflowParameter("sparkopts","--num-executors 3 --executor-memory 1G --executor-cores 3 --driver-memory 2G --files config.properties --conf spark.executor.extraJavaOptions=-XX:+UseConcMarkSweepGC" +" --conf spark.yarn.jar=hdfs://master01:8020/bobo/in/flow/spark-assembly-1.6.0-cdh5.9.0-hadoop2.6.0-cdh5.9.0.jar");WorkflowParameter jararg1 = new WorkflowParameter("jararg1","slave01:9092,slave02:9092,slave03:9092");WorkflowParameter jararg2 = new WorkflowParameter("jararg2","DATA-TOPIC");wfParameters.add(jobmaster);wfParameters.add(jobmode);wfParameters.add(jobname);wfParameters.add(jarclass);wfParameters.add(jarpath);wfParameters.add(sparkopts);wfParameters.add(jararg1);wfParameters.add(jararg2);// Start OozingString jobId = client.startJob(JOB_PATH, wfParameters);Status status = client.getJobStatus(jobId);if(status == Status.RUNNING)System.out.println("Workflow job running");elseSystem.out.println("Problem starting Workflow job");}
}4.workflow.xml的配置
<workflow-app name="Spark_Workflow" xmlns="uri:oozie:workflow:0.1"> <start to="spark-SparkOozieAction"/> <kill name="Kill"> <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> </kill> <action name="spark-SparkOozieAction"> <spark xmlns="uri:oozie:spark-action:0.1"> <job-tracker>${jobTracker}</job-tracker> <name-node>${nameNode}</name-node> <master>${jobmaster}</master> <mode>${jobmode}</mode> <name>${jobname}</name> <class>${jarclass}</class> <jar>${jarpath}</jar> <spark-opts>${sparkopts}</spark-opts> <arg>${jararg1}</arg> <arg>${jararg2}</arg> </spark> <ok to="End"/> <error to="Kill"/> </action> <end name="End"/>
</workflow-app> 这篇关于Java调用oozie提交spark on yarn任务的文章就介绍到这儿，希望我们推荐的文章对编程师们有所帮助！
