From 9c2b7a774d75d25901c98e235058b67e6c582e95 Mon Sep 17 00:00:00 2001 From: Altynbek Isabekov Date: Thu, 14 Nov 2024 21:04:13 +0100 Subject: [PATCH] Add an example for converting a map to a struct. --- pyspark_cookbook.org | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pyspark_cookbook.org b/pyspark_cookbook.org index 6228993..684d7ef 100644 --- a/pyspark_cookbook.org +++ b/pyspark_cookbook.org @@ -1321,6 +1321,33 @@ root |[1, 2, 5]|[1, 3]| [1, 3]| |[4, 4, 6]|[0, 2]| [0, 2]| +** To convert a map to a struct +#+BEGIN_SRC python :post pretty2orgtbl(data=*this*) + from pyspark.sql import SparkSession + import pyspark.sql.functions as F + spark = SparkSession.builder.master("local").appName("test-app").getOrCreate() + df = spark.sql("SELECT map('John', 1, 'Michael', 2) as Students_map") + df = df.withColumn("Students_struct", F.map_entries("Students_map").alias("a", "b")) + df.printSchema() + df.show(truncate=False) +#+END_SRC + +#+RESULTS: +:results: +root + |-- Students_map: map (nullable = false) + | |-- key: string + | |-- value: integer (valueContainsNull = false) + |-- Students_struct: array (nullable = false) + | |-- element: struct (containsNull = false) + | | |-- key: string (nullable = false) + | | |-- value: integer (nullable = false) + +| Students_map | Students_struct | +|---------------------------+---------------------------| +| {John -> 1, Michael -> 2} | [{John, 1}, {Michael, 2}] | +:end: + ** To slice an array #+BEGIN_SRC python :post pretty2orgtbl(data=*this*) import pyspark.sql.functions as F