Spark: Union Incompatible Dataframes

 1def incompat_union(df1: DataFrame, df2: DataFrame) -> DataFrame:
 2    """
 3    Union two incompatible DataFrames i.e. number of columns and order can be different.
 4    Creates a DataFrame which contains unique columns from both, and fills with nulls missing columns in each.
 5    :param df1:
 6    :param df2:
 7    """
 8
 9    # take first df and add missing columns, filling them with nulls
10    df1u = df1
11    for df2c in df2.columns:
12        if df2c not in df1.columns:
13            log.info(f"'{df2c}' not in df1, adding")
14            df1u = (df1u.withColumn(df2c, f.lit(None)))
15
16    # take df2 and make it idential in order and columns to df1u
17    cols2 = []
18    for df1c in df1u.columns:
19        if df1c in df2.columns:
20            # add own column
21            cols2.append(f.col(df1c))
22        else:
23            # add a dummy
24            cols2.append(f.lit(None).alias(df1c))
25            log.info(f"'{df1c}' not in df2, adding")
26    df2u = df2.select(*cols2)
27    df = df1u.union(df2u)
28
29    # df.printSchema()
30    return df