spark/sql/JavaDataFrameReaderWriterSuite.java

0001 /*
0002 * Licensed to the Apache Software Foundation (ASF) under one or more
0003 * contributor license agreements.  See the NOTICE file distributed with
0004 * this work for additional information regarding copyright ownership.
0005 * The ASF licenses this file to You under the Apache License, Version 2.0
0006 * (the "License"); you may not use this file except in compliance with
0007 * the License.  You may obtain a copy of the License at
0008 *
0009 *    http://www.apache.org/licenses/LICENSE-2.0
0010 *
0011 * Unless required by applicable law or agreed to in writing, software
0012 * distributed under the License is distributed on an "AS IS" BASIS,
0013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014 * See the License for the specific language governing permissions and
0015 * limitations under the License.
0016 */
0017
0018 package test.org.apache.spark.sql;
0019
0020 import java.io.File;
0021 import java.util.HashMap;
0022
0023 import org.apache.spark.sql.SaveMode;
0024 import org.apache.spark.sql.SparkSession;
0025 import org.apache.spark.sql.test.TestSparkSession;
0026 import org.apache.spark.sql.types.StructType;
0027 import org.apache.spark.util.Utils;
0028 import org.junit.After;
0029 import org.junit.Before;
0030 import org.junit.Test;
0031
0032 public class JavaDataFrameReaderWriterSuite {
0033   private SparkSession spark = new TestSparkSession();
0034   private StructType schema = new StructType().add("s", "string");
0035   private transient String input;
0036   private transient String output;
0037
0038   @Before
0039   public void setUp() {
0040     input = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "input").toString();
0041     File f = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "output");
0042     f.delete();
0043     output = f.toString();
0044   }
0045
0046   @After
0047   public void tearDown() {
0048     spark.stop();
0049     spark = null;
0050   }
0051
0052   @Test
0053   public void testFormatAPI() {
0054     spark
0055         .read()
0056         .format("org.apache.spark.sql.test")
0057         .load()
0058         .write()
0059         .format("org.apache.spark.sql.test")
0060         .save();
0061   }
0062
0063   @Test
0064   public void testOptionsAPI() {
0065     HashMap<String, String> map = new HashMap<>();
0066     map.put("e", "1");
0067     spark
0068         .read()
0069         .option("a", "1")
0070         .option("b", 1)
0071         .option("c", 1.0)
0072         .option("d", true)
0073         .options(map)
0074         .text()
0075         .write()
0076         .option("a", "1")
0077         .option("b", 1)
0078         .option("c", 1.0)
0079         .option("d", true)
0080         .options(map)
0081         .format("org.apache.spark.sql.test")
0082         .save();
0083   }
0084
0085   @Test
0086   public void testSaveModeAPI() {
0087     spark
0088         .range(10)
0089         .write()
0090         .format("org.apache.spark.sql.test")
0091         .mode(SaveMode.ErrorIfExists)
0092         .save();
0093   }
0094
0095   @Test
0096   public void testLoadAPI() {
0097     spark.read().format("org.apache.spark.sql.test").load();
0098     spark.read().format("org.apache.spark.sql.test").load(input);
0099     spark.read().format("org.apache.spark.sql.test").load(input, input, input);
0100     spark.read().format("org.apache.spark.sql.test").load(new String[]{input, input});
0101   }
0102
0103   @Test
0104   public void testTextAPI() {
0105     spark.read().text();
0106     spark.read().text(input);
0107     spark.read().text(input, input, input);
0108     spark.read().text(new String[]{input, input})
0109         .write().text(output);
0110   }
0111
0112   @Test
0113   public void testTextFileAPI() {
0114     spark.read().textFile();
0115     spark.read().textFile(input);
0116     spark.read().textFile(input, input, input);
0117     spark.read().textFile(new String[]{input, input});
0118   }
0119
0120   @Test
0121   public void testCsvAPI() {
0122     spark.read().schema(schema).csv();
0123     spark.read().schema(schema).csv(input);
0124     spark.read().schema(schema).csv(input, input, input);
0125     spark.read().schema(schema).csv(new String[]{input, input})
0126         .write().csv(output);
0127   }
0128
0129   @Test
0130   public void testJsonAPI() {
0131     spark.read().schema(schema).json();
0132     spark.read().schema(schema).json(input);
0133     spark.read().schema(schema).json(input, input, input);
0134     spark.read().schema(schema).json(new String[]{input, input})
0135         .write().json(output);
0136   }
0137
0138   @Test
0139   public void testParquetAPI() {
0140     spark.read().schema(schema).parquet();
0141     spark.read().schema(schema).parquet(input);
0142     spark.read().schema(schema).parquet(input, input, input);
0143     spark.read().schema(schema).parquet(new String[] { input, input })
0144         .write().parquet(output);
0145   }
0146
0147   /**
0148    * This only tests whether API compiles, but does not run it as orc()
0149    * cannot be run without Hive classes.
0150    */
0151   public void testOrcAPI() {
0152     spark.read().schema(schema).orc();
0153     spark.read().schema(schema).orc(input);
0154     spark.read().schema(schema).orc(input, input, input);
0155     spark.read().schema(schema).orc(new String[]{input, input})
0156         .write().orc(output);
0157   }
0158 }