大数据_搭建SPARK开发环境

1. 项目结构设置

1.1 项目设置

1.2 SDK设置

1.3 全局库设置

2. 新建项目

3. 配置文件

settings.xml 加入新的mirror

-<mirror>

<id>alimaven</id>

<mirrorOf>central</mirrorOf>

<name>aliyun maven</name>

<url>https://maven.aliyun.com/repository/central</url>

</mirror>

pom.xml，版本根据对应的修改

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

  <modelVersion>4.0.0</modelVersion>

  <groupId>com.xtd.spark</groupId>
  <artifactId>spark</artifactId>
  <packaging>jar</packaging>
  <version>1.0</version>

  <name>spark scala maven</name>

  <!--声明共有的属性-->
  <properties>
    <maven.compiler.source>1.8</maven.compiler.source>
    <maven.compiler.target>1.8</maven.compiler.target>
    <encoding>UTF-8</encoding>
    <scala.version>2.11.8</scala.version>
    <hadoop.version>3.2.0</hadoop.version>
    <spark.version>2.2.0</spark.version>
    <scala.compat.version>2.11</scala.compat.version>
  </properties>
  <!--声明并引入公有的依赖-->
  <dependencies>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
      <version>${scala.version}</version>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.11</artifactId>
      <version>${spark.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_2.11</artifactId>
      <version>${spark.version}</version>
    </dependency>

    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client</artifactId>
      <version>${hadoop.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-hive_2.11</artifactId>
      <version>${spark.version}</version>
      <!--<scope>provided</scope>-->
    </dependency>

  </dependencies>

  <!--配置构建信息-->
  <build>
    <!--资源文件夹-->
    <sourceDirectory>src/main/scala</sourceDirectory>
    <!--声明并引入构建的插件-->
    <plugins>
      <!--用于编译Scala代码到class-->
      <plugin>
        <groupId>net.alchim31.maven</groupId>
        <artifactId>scala-maven-plugin</artifactId>
        <version>3.2.2</version>
        <executions>
          <execution>
            <goals>
              <goal>compile</goal>
              <goal>testCompile</goal>
            </goals>
            <configuration>
              <args>
                <arg>-dependencyfile</arg>
                <arg>${project.build.directory}/.scala_dependencies</arg>
              </args>
            </configuration>
          </execution>
        </executions>
      </plugin>

    </plugins>
  </build>


</project>

4. 编译程序