当前位置：首页 > article >正文

springboot远程链接spark

article 2025/1/9 20:00:58

springboot远程链接spark

1、导入依赖

<!--        spark依赖-->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.12</artifactId>
            <version>3.2.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.12</artifactId>
            <version>3.2.2</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-mllib -->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-mllib_2.12</artifactId>
            <version>3.2.2</version>
        </dependency>

2、配置spark信息

建立一个配置文件，配置spark信息

import org.apache.spark.SparkConf;
import org.apache.spark.sql.SparkSession;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

//将文件交于spring管理
@Configuration
public class SparkConfig {

    //使用yml中的配置
    @Value("${spark.master}")
    private String sparkMaster;

    @Value("${spark.appName}")
    private String sparkAppName;
    @Bean
    public SparkConf sparkConf() {
        SparkConf conf = new SparkConf();
        conf.setMaster(sparkMaster);
        conf.setAppName(sparkAppName);
        return conf;
    }

    @Bean
    public SparkSession sparkSession() {
        return SparkSession.builder()
                .config(sparkConf())
                .getOrCreate();
    }
}

3、controller和service

controller类

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import xyz.zzj.traffic_main_code.service.SparkService;

@RestController
@RequestMapping("/spark")
public class SparkController {

    @Autowired
    private SparkService sparkService;

    @GetMapping("/run")
    public String runSparkJob() {
        sparkService.executeSparkJob();
        return "Spark job executed successfully!";
    }
}

service

import org.springframework.stereotype.Service;

@Service
public class SparkService {

    public void executeSparkJob() {
        System.out.println("Spark job started");
    }
}