diff --git a/pyspark_cookbook.org b/pyspark_cookbook.org index 6228993..85aca28 100644 --- a/pyspark_cookbook.org +++ b/pyspark_cookbook.org @@ -1321,6 +1321,33 @@ root |[1, 2, 5]|[1, 3]| [1, 3]| |[4, 4, 6]|[0, 2]| [0, 2]| +** To convert a map to a struct +#+BEGIN_SRC python :post pretty2orgtbl(data=*this*) + from pyspark.sql import SparkSession + import pyspark.sql.functions as F + spark = SparkSession.builder.master("local").appName("test-app").getOrCreate() + df = spark.sql("SELECT map('John', 1, 'Michael', 2) as Students_map") + df = df.withColumn("Students_struct", F.map_entries("Students_map").alias("a", "b")) + df.printSchema() + df.show(truncate=False) +#+END_SRC + +#+RESULTS: +:results: +root + |-- Students_map: map (nullable = false) + | |-- key: string + | |-- value: integer (valueContainsNull = false) + |-- Students_struct: array (nullable = false) + | |-- element: struct (containsNull = false) + | | |-- key: string (nullable = false) + | | |-- value: integer (nullable = false) + +|Students_map |Students_struct | +|-------------------------+-------------------------| +|{John -> 1, Michael -> 2}|[{John, 1}, {Michael, 2}]| +:end: + ** To slice an array #+BEGIN_SRC python :post pretty2orgtbl(data=*this*) import pyspark.sql.functions as F