mini-lakehouse-lab/notebooks/spark-basic-test.ipynb

In [1]:

from pyspark.sql import SparkSession

spark = (
    SparkSession.builder
    .appName("check")
    .master("spark://spark-master:7077")
    .getOrCreate()
)

spark.sql("""
    CREATE TABLE IF NOT EXISTS lakehouse.default.demo_fix (
        id BIGINT,
        txt STRING
    )
    USING iceberg
""")

spark.sql("INSERT INTO lakehouse.default.demo_fix VALUES (1, 'ok')")

spark.sql("SELECT * FROM lakehouse.default.demo_fix").show()

Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
25/12/03 09:08:48 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
25/12/03 09:08:52 WARN MetricsConfig: Cannot locate configuration: tried hadoop-metrics2-s3a-file-system.properties,hadoop-metrics2.properties
                                                                                

+---+---+
| id|txt|
+---+---+
|  1| ok|
+---+---+

In [ ]: