JUNIT 测试用例-Spark JDBC

Question

我是 Java 编程的新手。我有一个从Oracle数据库读取数据的方法。现在我需要帮助使用 JUnit 框架为以下代码编写测试用例。

数据集 df = Spark.read().format("jdbc").jdbc(jdbcUrl, dbTable1, connectionProperties);

Answer 1

您可以使用以下方法，您需要使用 H2 库来创建 带有数据库的内存临时服务器，您可以在其中创建所需的表 测试数据。此示例使用 MS Sql Server (MODE=MSSQLServer)，您可以更改它。

    <!-- https://mvnrepository.com/artifact/com.h2database/h2 -->
    <dependency>
      <groupId>com.h2database</groupId>
      <artifactId>h2</artifactId>
      <version>1.4.200</version>
      <scope>test</scope>
    </dependency>

实际方法

    def GetJDBCDataframe(spark: SparkSession, jdbcUrl: String, connectionProperties :Properties): DataFrame = {
        val employeesQuery = "(SELECT * FROM Employee) ref_alias"
        val df = spark.read.jdbc(url = jdbcUrl, table = employeesQuery, connectionProperties)
        df
    }

测试用例方法：

    @Test
    def GetJDBCDataframe(): Unit = {
    
      val spark: SparkSession = {
        SparkSession
          .builder()
          .master("local[2]")
          .appName("SampleSparkScalaTest")
          .getOrCreate()
      }
      //H2 library connection for creating a inmemory MSSQL databse
      val url = "jdbc:h2:mem:Mydatabasename;MODE=MSSQLServer"
    
      val connectionProperties = new Properties()
      connectionProperties.setProperty("user", "sa")
      connectionProperties.setProperty("password", "")
    
      Class.forName("org.h2.Driver")
      val conn = DriverManager.getConnection(url, connectionProperties)
    
      //create mock jdbc Employee table (immemory)
      conn.prepareStatement("create table Employee(EmpId int, FirstName varchar, LastName varchar)").executeUpdate()
      conn.prepareStatement("insert into Employee values (1, 'Brain', 'Lara')").executeUpdate()
      conn.prepareStatement("insert into Employee values (2, 'Virat', 'Kholi')").executeUpdate()
      conn.prepareStatement("insert into Employee values (3, 'MS', 'Dhoni')").executeUpdate()
      conn.commit()
    
      val rowCount = TransactionLogicTransformation.GetJDBCDataframe(spark,url,connectionProperties).count()
    
      assertEquals(3, rowCount)
    }