первый commit

src/spark/cluster_smoke.py

@@ -0,0 +1,12 @@
+from pyspark.sql import SparkSession
+
+spark = (
+    SparkSession.builder
+    .appName("test-cluster")
+    .master("spark://spark-master:7077")  # важно: не local[*]
+    .getOrCreate()
+)
+
+spark.range(0, 1000000).groupBy().sum().show()
+
+spark.stop()

src/spark/iceberg_demo.sql

@@ -0,0 +1,10 @@
+-- создаём таблицу в каталоге lakehouse, в MinIO
+CREATE TABLE lakehouse.default.demo_tbl (
+  id BIGINT,
+  txt STRING
+)
+USING iceberg;
+
+INSERT INTO lakehouse.default.demo_tbl VALUES (1, 'hello'), (2, 'world');
+
+SELECT * FROM lakehouse.default.demo_tbl;

src/spark/iceberg_smoke.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+from pyspark.sql import SparkSession
+
+
+TABLE = "lakehouse.default.spark_trino_smoke"
+
+
+def main() -> None:
+    spark = SparkSession.builder.appName("iceberg-smoke").getOrCreate()
+
+    spark.sql("CREATE NAMESPACE IF NOT EXISTS lakehouse.default")
+    spark.sql(f"DROP TABLE IF EXISTS {TABLE}")
+    spark.sql(f"CREATE TABLE {TABLE} (id INT, payload STRING) USING iceberg")
+    spark.sql(f"INSERT INTO {TABLE} VALUES (1, 'from_spark')")
+
+    result = spark.sql(f"SELECT * FROM {TABLE} ORDER BY id")
+    result.show(truncate=False)
+    print(f"✔ Spark записал и прочитал таблицу {TABLE}")
+
+    spark.stop()
+
+
+if __name__ == "__main__":
+    main()

src/spark/partitioned_table_demo.sql

@@ -0,0 +1,20 @@
+-- Партиционированная Iceberg-таблица lakehouse.default.partition_demo
+
+CREATE NAMESPACE IF NOT EXISTS lakehouse.default;
+
+DROP TABLE IF EXISTS lakehouse.default.partition_demo;
+
+CREATE TABLE lakehouse.default.partition_demo (
+  user_id    BIGINT,
+  event_date DATE,
+  amount     DOUBLE
+)
+USING iceberg
+PARTITIONED BY (event_date);
+
+INSERT INTO lakehouse.default.partition_demo VALUES
+  (1, '2024-01-01', 10.0),
+  (2, '2024-01-01', 20.0),
+  (3, '2024-02-01', 30.0),
+  (4, '2024-02-01', 40.0);
+

src/spark/schema_evolution_demo.sql

@@ -0,0 +1,23 @@
+-- Эволюция схемы Iceberg-таблицы lakehouse.default.schema_evolution_demo
+
+CREATE NAMESPACE IF NOT EXISTS lakehouse.default;
+
+DROP TABLE IF EXISTS lakehouse.default.schema_evolution_demo;
+
+CREATE TABLE lakehouse.default.schema_evolution_demo (
+  id    BIGINT,
+  value STRING
+)
+USING iceberg;
+
+INSERT INTO lakehouse.default.schema_evolution_demo VALUES
+  (1, 'old_row_1'),
+  (2, 'old_row_2');
+
+ALTER TABLE lakehouse.default.schema_evolution_demo
+  ADD COLUMN metadata STRING;
+
+INSERT INTO lakehouse.default.schema_evolution_demo VALUES
+  (3, 'new_row_1', 'extra_info'),
+  (4, 'new_row_2', 'extra_info');
+

src/trino/iceberg_smoke.sql

@@ -0,0 +1,5 @@
+-- Проверка чтения данных, созданных в Spark
+USE lakehouse.default;
+
+SHOW TABLES;
+SELECT * FROM spark_trino_smoke;

src/trino/schema_evolution_demo.sql

@@ -0,0 +1,13 @@
+-- Проверка эволюции схемы в Trino для таблицы,
+-- созданной в Spark (lakehouse.default.schema_evolution_demo).
+
+USE lakehouse.default;
+
+SHOW TABLES LIKE 'schema_evolution_demo';
+
+DESCRIBE schema_evolution_demo;
+
+SELECT *
+FROM schema_evolution_demo
+ORDER BY id;
+