Working with Spark on R Studio

1# connect to Databricks on this server
2
3library(SparkR)
4SparkR::sparkR.session()
5
6library(sparklyr)
7sc <- spark_connect(method = "databricks")
8
9library(dplyr)

1tbl_change_db(sc, "mydb") #change active db to "mydb" from "default"
2src_tbls(sc) # list tables

1spark_df <- spark_read_table(sc, "mytable") # read as Spark DataFrame
2r_df <- collect(spark_df) # convert to R DataFrame

 1tbl_change_db(sc, "default") # changing db to "default" - where  we want to write to
 2# copy data into Spark DataFrame
 3t <- copy_to(sc,
 4             r_df,
 5             "mytable",
 6             overwrite = TRUE)
 7# save as table in Spark
 8spark_write_table(t, "mytable", mode = "overwrite")
 9# list tables
10src_tbls(sc)