本文共 3076 字，大约阅读时间需要 10 分钟。

一、依赖包配置

scala与spark的相关依赖包，spark包后尾下划线的版本数字要跟scala的版本第一二位要一致，即2.11

pom.xml

   
     
    
     4.0.0
      
    
     com.mk
      
    
     spark-test
      
    
     1.0
      
    
     spark-test
      
    
     http://spark.mk.com
      
        
     
      UTF-8
         
     
      1.8
         
     
      1.8
         
     
      2.11.1
         
     
      2.4.4
         
     
      2.6.0
       
      
        
         
           
      
       org.scala-lang
            
      
       scala-library
            
      
       ${scala.version}
          
         
         
           
      
       org.apache.spark
            
      
       spark-core_2.11
            
      
       ${spark.version}
          
         
           
      
       org.apache.spark
            
      
       spark-sql_2.11
            
      
       ${spark.version}
          
         
           
      
       junit
            
      
       junit
            
      
       4.11
            
      
       test
          
       
      
        
           
              
                 
        
         maven-clean-plugin
                  
        
         3.1.0
                
               
                 
        
         maven-resources-plugin
                  
        
         3.0.2
                
               
                 
        
         maven-compiler-plugin
                  
        
         3.8.0
                
               
                 
        
         maven-surefire-plugin
                  
        
         2.22.1
                
               
                 
        
         maven-jar-plugin
                  
        
         3.0.2
                
             
          
       
    
   

二、PI例子

java重新编写scala的PI例子

package com.mk;import org.apache.spark.SparkConf;import org.apache.spark.api.java.JavaSparkContext;import org.apache.spark.sql.SparkSession;import java.util.ArrayList;import java.util.List;public class App {    public static void main( String[] args )    {        SparkConf sparkConf = new SparkConf();        if(System.getProperty("os.name").toLowerCase().contains("win")) {            sparkConf.setMaster("local[2]");//本地模拟            System.out.println("使用本地模拟是spark");        }        SparkSession session = SparkSession.builder().appName("Pi").config(sparkConf).config(sparkConf).getOrCreate();        int slices =2;        int n = (int)Math.min(100_000L * slices, Integer.MAX_VALUE);        JavaSparkContext sparkContext = new JavaSparkContext(session.sparkContext());        List
   
     list = new ArrayList<>(n);        for (int i = 0; i < n; i++)            list.add(i + 1);        int count  = sparkContext.parallelize(list, slices).                map(v -> {                    double x = Math.random() * 2 - 1;                    double y = Math.random() * 2 - 1;                    if (x * x + y * y < 1)                        return 1;                    return 0;                }).reduce((Integer a, Integer b) ->a+b);         System.out.println("PI:"+  4.0 * count / n);        session.stop();    }}
   

三、直接在idea本地运行

输出PI

四、spark集群提交

项目打成jar，把spark-test.jar上传到~目录，执行shell命令

~/software/spark-2.4.4-bin-hadoop2.6/bin/spark-submit --master spark://hadoop01:7077,hadoop02:7077,hadoop03:7077 --class com.mk.App ~/spark-test.jar

转载地址：https://maokun.blog.csdn.net/article/details/104119225 如侵犯您的版权，请留言回复原文章的地址，我们会给您删除此文章，给您带来不便请您谅解！