Spark SQL Dataframe Doris的输入输出操作

需要的maven依赖

<properties>
    <maven.compiler.source>8</maven.compiler.source>
    <maven.compiler.target>8</maven.compiler.target>
    <scala.binary.version>2.12</scala.binary.version>
    <scala.version>2.12.15</scala.version>
    <spark.version>3.2.4</spark.version>
    <commons.math3.version>3.4.1</commons.math3.version>
    <scalacheck.version>1.15.4</scalacheck.version>
</properties>

<dependencies>
    <dependency>
        <groupId>org.apache.doris</groupId>
        <artifactId>spark-doris-connector-3.2_2.12</artifactId>
        <version>1.1.0</version>
        <exclusions>
            <exclusion>
                <groupId>org.apache.spark</groupId>
                <artifactId>*</artifactId>
            </exclusion>
        </exclusions>
    </dependency>
    <!-- spark -->
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-core_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <scope>provided</scope>
    </dependency>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-sql_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <scope>provided</scope>
    </dependency>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-hive_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <scope>provided</scope>
        <exclusions>
            <exclusion>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>*</artifactId>
            </exclusion>
        </exclusions>
    </dependency>
</dependencies>

Spark 读取doris数据

val session: SparkSession = SparkSession.builder()
  .master("local[4]")
  .appName(getClass.getSimpleName)
  .enableHiveSupport()
  .getOrCreate()

session.read
  .format("doris")
  .option("doris.table.identifier", "doris表名")
  .option("doris.fenodes", "doris Fe地址")
  .option("user", "用户名")
  .option("password", "密码")
  .load()
  .show()

session.stop()

Spark 写入Doris数据

val session: SparkSession = SparkSession.builder()
  .master("local[4]")
  .appName(getClass.getSimpleName)
  .getOrCreate()

import session.{catalog, table}

dataframe
  .write
  .format("doris")
  .option("doris.table.identifier", "doris表名") 
  .option("doris.fenodes", "doris Fe地址") 
  .option("user", "用户名") .option("password", "密码")
  .save()

session.stop()