docs(scratch): handoff — триаж #5 закрыт, ревью APPROVED, идёт приёмка

- Зачем: - зафиксировать состояние конвейера #5 перед долгой живой приёмкой, чтобы новая сессия продолжила без потери контекста. - Что: - в handoff добавлена дельта 23:10: обе находки FIXED (фрагменты ID с SHA-256-цепочкой), перепроверка линией B — APPROVED; - обновлено состояние стенда: им владеет сценарий приёмки; - в .scratch/issue5-run добавлены свежие отчёты, перепроверка и сценарий приёмки acceptance-issue5.sh. - Проверка: - git show --stat; лог приёмки — до строки SCRIPT_EXIT_CODE=.
2026-07-22 23:11:44 +03:00
parent ceb30a95c0
commit f4971e94ca
10 changed files with 803 additions and 3 deletions
@@ -0,0 +1,135 @@
+#!/bin/bash
+# Live acceptance for issue #5: incremental manifest counters.
+# Focus: next-day runs must be O(new day) — flat wall time across days,
+# no full Kafka reread; chain-check green on the grown world.
+# UI actions emulated via airflow CLI in the scheduler container
+# (session-auth-only REST on this stand; trigger with no conf == empty form).
+set -u
+cd /home/dementev/sources/clickstream-ch-kafka-superset-demo || exit 1
+
+AF() { docker compose exec -T airflow-scheduler airflow "$@"; }
+CH_QUERY() { docker compose exec -T clickhouse clickhouse-client --user=default --password=123456 --query "$1"; }
+
+step() { echo; echo "=== [$(date +%H:%M:%S)] $*"; }
+fail() { echo "!!! ACCEPTANCE FAILED: $*"; exit 1; }
+
+latest_run_state() { # dag_id -> "run_id state" of newest run
+  AF dags list-runs -d "$1" -o json 2>/dev/null | python3 -c '
+import sys, json
+rs = json.load(sys.stdin)
+rs.sort(key=lambda r: r["execution_date"], reverse=True)
+print(rs[0]["run_id"], rs[0]["state"]) if rs else print("none", "none")'
+}
+
+runs_count() {
+  AF dags list-runs -d "$1" -o json 2>/dev/null | python3 -c 'import sys,json;print(len(json.load(sys.stdin)))'
+}
+
+wait_dag_done() { # dag_id timeout_sec
+  local dag="$1" t="$2" line="" state=""
+  local deadline=$(( $(date +%s) + t ))
+  while [ "$(date +%s)" -lt "$deadline" ]; do
+    line=$(latest_run_state "$dag"); state="${line##* }"
+    case "$state" in
+      success) echo "$dag ${line% *}: success"; return 0 ;;
+      failed)  echo "$dag ${line% *}: FAILED"; return 1 ;;
+      *) sleep 20 ;;
+    esac
+  done
+  echo "$dag: TIMEOUT (last: $line)"; return 1
+}
+
+run_durations() { # dag_id -> per-run "run_id duration_s" sorted by exec date
+  AF dags list-runs -d "$1" -o json 2>/dev/null | python3 -c '
+import sys, json, datetime as dt
+def p(s): return dt.datetime.fromisoformat(s.replace("Z", "+00:00"))
+rs = json.load(sys.stdin)
+rs.sort(key=lambda r: r["execution_date"])
+for r in rs:
+    if r.get("start_date") and r.get("end_date"):
+        d = (p(r["end_date"]) - p(r["start_date"])).total_seconds()
+        print(f"{r[\"run_id\"]} {r[\"state\"]} {d:.0f}s")
+    else:
+        print(f"{r[\"run_id\"]} {r[\"state\"]} -")'
+}
+
+world_stats() {
+  CH_QUERY "SELECT uniqExact(event_date) AS days, count() AS events, uniqExact(user_domain_id) AS users FROM dm.v_events_enriched FORMAT TSVWithNames"
+}
+
+# --- 0. Pre-flight ---
+step "Pre-flight"
+[ -S /var/run/docker.sock ] || fail "docker socket absent"
+docker info >/dev/null 2>&1 || fail "docker daemon not answering"
+echo "docker OK"
+
+# --- 1. Clean stand + up ---
+step "make clean (wipe volumes)"
+make clean || fail "make clean"
+step "make up"
+make up || fail "make up"
+
+step "wait for Airflow (scheduler parsed DAGs, webserver answers)"
+ok=""
+for i in $(seq 1 60); do
+  if AF dags list -o plain 2>/dev/null | grep -q "world_init"; then ok=1; break; fi
+  sleep 10
+done
+[ -n "$ok" ] || fail "world_init not parsed after 10 min"
+ok=""
+for i in $(seq 1 18); do
+  if curl -sf -o /dev/null http://localhost:8080/login/; then ok=1; break; fi
+  sleep 10
+done
+[ -n "$ok" ] || fail "webserver UI not answering after 3 min"
+echo "Airflow up"
+
+# --- 2. Ladder: ddl_init, etl, world_init (empty form => import) ---
+step "ddl_init: unpause + trigger"
+AF dags unpause ddl_init >/dev/null || fail "unpause ddl_init"
+AF dags trigger ddl_init >/dev/null || fail "trigger ddl_init"
+wait_dag_done ddl_init 600 || fail "ddl_init run"
+
+step "unpause etl_pipeline; world_init trigger (empty form)"
+AF dags unpause etl_pipeline >/dev/null || fail "unpause etl_pipeline"
+AF dags unpause world_init >/dev/null || fail "unpause world_init"
+t0=$(date +%s)
+AF dags trigger world_init >/dev/null || fail "trigger world_init"
+wait_dag_done world_init 2400 || fail "world_init run (import)"
+echo "IMPORT_WALL_TIME=$(( $(date +%s) - t0 ))s"
+
+step "world after import"
+world_stats || fail "dm query after import"
+
+# --- 3. Three next-day runs, each timed (core of #5) ---
+# Unpause => catchup=False gives run 1 immediately; runs 2-3 by manual
+# trigger (same empty-form semantics). max_active_runs=1 serializes any
+# stray :30 boundary run; durations are read from run metadata per run.
+step "world_next_day: unpause (run 1)"
+AF dags unpause world_next_day >/dev/null || fail "unpause world_next_day"
+sleep 60
+[ "$(runs_count world_next_day)" -ge 1 ] || fail "no immediate run after unpause"
+wait_dag_done world_next_day 3600 || fail "next-day run 1"
+
+for i in 2 3; do
+  step "world_next_day: manual trigger (run $i)"
+  AF dags trigger world_next_day >/dev/null || fail "trigger run $i"
+  sleep 20
+  wait_dag_done world_next_day 3600 || fail "next-day run $i"
+done
+
+step "re-pause world_next_day"
+AF dags pause world_next_day >/dev/null || fail "re-pause"
+
+step "next-day run durations (must be roughly flat: O(new day))"
+run_durations world_next_day
+
+step "world after 3 next days"
+world_stats || fail "dm query after next days"
+
+# --- 4. Chain check on the grown world ---
+step "make generated-history-chain-check"
+make generated-history-chain-check || fail "chain-check"
+
+echo
+echo "=== ACCEPTANCE PASSED [$(date +%H:%M:%S)] ==="
@@ -0,0 +1,94 @@
+# Mandate: issue #5 — incremental manifest counters (next-day without full Kafka reread)
+
+Repo: /home/dementev/sources/clickstream-ch-kafka-superset-demo
+Branch: feature/mentee-path (work in place, do NOT commit).
+Python via uv. Docs/comments/code text that stays in repo: clear Russian.
+This exchange file and your report: English.
+
+## Source of truth (read first, in this order)
+
+1. Issue #5 criteria — reproduced below (GitHub CLI may not work in your sandbox).
+2. Spec: `docs/specs/2026-07-19-mentee-path-redesign.md` — «Решения» item 4,
+   «Чего здесь не делаем».
+3. `.scratch/hitl-findings.md`, finding F9 — the code-level analysis of WHERE
+   the full Kafka reread happens. This is your map into the code.
+
+## Goal
+
+The scheduled `next-day` stops getting more expensive as the world ages.
+Today every next-day run rereads the ENTIRE Kafka history to rebuild manifest
+counters: cost grows ~quadratically over days (measured: day 2 — 638 s),
+memory grows linearly (OOM risk).
+
+## Acceptance criteria (from issue #5)
+
+- Cumulative counter state (sums, uid/click_id sets, rolling checksum) is
+  stored in state/manifest.
+- A new day is append-only: NO full reread of Kafka history remains in the
+  next-day path.
+- Generation+manifest task on day N takes ~the same time as day 1.
+- `make generated-history-chain-check` green on day seams.
+- `make test` and `make lint` green.
+
+## Architecture guidance (coordinator's preferred direction)
+
+Open question you must resolve and JUSTIFY in the report: how cumulative
+counters get seeded when the world comes from the reference artifact in git
+(`data/startup_history/reference-world.json.xz`), whose stored state predates
+the new counters.
+
+Preferred direction: seed the cumulative counters wherever the event stream
+already flows through the generator — i.e. during `import` (artifact is fully
+read to load Kafka anyway) and during `backfill` (events are produced by us).
+Then `next-day` only updates counters incrementally. The artifact format in
+git should NOT need to change; only the locally-stored state/manifest gains
+the cumulative section. If state lacks the cumulative section (old local
+state), fail with a clear Russian error telling the user to re-run import —
+do not silently fall back to a full reread.
+
+If you find this infeasible or substantially worse than an alternative, STOP
+and write your reasoning + alternative to the report file, then return with a
+question instead of implementing something else.
+
+Determinism invariants (must hold):
+- single-threaded PRNG usage unchanged; do NOT parallelize generation;
+- for the same world, manifest numbers (events, visits, users, checksum)
+  must be identical whether computed the old way (full read) or via the new
+  incremental path — the rolling checksum design must guarantee this
+  (e.g. order-independent combine or strictly deterministic order);
+- `check` operation semantics preserved: it must still be able to verify
+  ClickHouse against the manifest.
+
+## Boundaries
+
+- Do NOT parallelize generation (breaks determinism — single PRNG stream,
+  visits crossing day boundaries).
+- Do NOT do incremental ETL: world_next_day keeps calling etl_pipeline
+  full_refresh (tech debt tracked separately in #8).
+- Do NOT touch retention beyond the incremental counters.
+- Do NOT change the reference artifact file in git.
+- Do NOT touch `docs/course/`.
+- Do NOT commit.
+- Update docs affected by the state/manifest change in the same change
+  (at minimum `docs/runbooks/startup-history.md` if state format/semantics
+  visible there; check `docs/ARCHITECTURE.md`, `docs/OPERATIONS.md`).
+
+## Verification (run before reporting done)
+
+- `make test` and `make lint` green (exit codes in report).
+- `make generated-history-chain-check` if runnable in your environment;
+  if it needs the live stand and you cannot run it, say so explicitly —
+  the coordinator will run it during live acceptance.
+- Unit tests for the new counter state: seeding on import/backfill,
+  incremental update on next-day, equality of incremental vs full-recompute
+  manifest numbers on a small fixture (this equality test is REQUIRED).
+- Follow repo data rules: tests use small slices, not full jsonl files.
+
+## Report
+
+Write full report to
+`/tmp/claude-1000/-home-dementev-sources-clickstream-ch-kafka-superset-demo/0836e970-ee07-4f86-a61f-291f32da525a/scratchpad/report-issue5.md`:
+design decision on seeding + why, what changed (file list), how the rolling
+checksum stays equal to the full recompute, test/lint exit codes and
+counters, anything NOT done or uncertain. Return only a short summary + the
+report path.
@@ -0,0 +1,70 @@
+# Narrow recheck: issue #5 fix round (findings CODE-1, CODE-2 only)
+
+You are review line B (code quality) doing a NARROW recheck after a fix
+round. Fresh session, no executor history. Do NOT modify sources, git
+state, or permanent environment. You may read anything and run focused
+tests (pytest of specific files is fine).
+
+Repo: /home/dementev/sources/clickstream-ch-kafka-superset-demo, branch
+feature/mentee-path. Scope: ALL uncommitted working-tree changes
+(git status / git diff vs HEAD), but ONLY re-verify the two findings below.
+This is NOT a full re-review — do not open new lines of critique unless
+you find a defect introduced BY the fix itself.
+
+Context files (same directory as this file):
+- review-issue5-code.md — your line's original full report (CODE-1..CODE-2)
+- report-issue5.md — executor's report; section "## Triage round 1"
+  describes the fix design and measurements
+- mandate-issue5.md — original executor mandate (invariants)
+
+## Finding CODE-1 (was High, == TASK-1): manifest payload ceiling
+
+Original defect: cumulative manifest embedded exact click_id/uid sets,
+~0.81 MB already on the reference world, Kafka 1 MiB default message limit
+not overridden → save fails after ~1 more next-day; storage O(history) in
+a SINGLE message.
+
+Executor's claimed fix: exact ID sets moved out of the manifest into
+content-addressed chunks (≤10,000 IDs each, SHA-256 chain, separate compact
+topic generator_startup_history_counter_chunks); main manifest keeps totals
+ rolling checksums + chain head/count; next-day appends only new-ID
+chunks, never rewrites or scans history; explicit 900,000-byte pre-send
+guard with Russian error on both manifest and chunks.
+
+Verify:
+1. The single-message ceiling is genuinely gone: no code path serializes
+   the full ID history into one Kafka message (check import, backfill AND
+   next-day paths).
+2. Chunk chain integrity: hash chain actually validated on read; chain
+   head/count in manifest actually checked; a broken/missing chunk fails
+   loudly (Russian error), not silently.
+3. next-day is still O(new day): no full chunk-chain scan or historical
+   chunk rewrite on the incremental path.
+4. Exactness preserved: uniqExact comparison against ClickHouse still uses
+   exact counts; the required incremental-vs-full-recompute equality test
+   still genuinely compares incremental vs full recompute after the
+   restructure (not weakened to pass).
+5. The 900,000-byte guard is real (applied before send on every producer
+   path for these payloads) and its failure mode is explicit.
+6. Boundary-ID sourcing from generator state (active visits + users with
+   visits, pre-created unused users excluded) does not break determinism
+   or the day-boundary duplicate handling the tests claim.
+
+## Finding CODE-2 (was Low): write ordering
+
+Original defect: next-day saved state BEFORE manifest (import did reverse),
+widening the half-advanced window.
+
+Verify: order is now chunks → manifest → state on ALL three paths (import,
+backfill, next-day), and the two-day test actually asserts it.
+
+## Report
+
+Write full report (English) to recheck-issue5-code.md in this same
+directory. Structure: per finding — RESOLVED / NOT_RESOLVED / REGRESSION
+(new defect introduced by the fix, with severity, file:line, scenario).
+End with VERDICT: APPROVED (both resolved, no regressions) or
+CHANGES_REQUESTED.
+
+Final message: VERDICT, one line per finding status, one line per
+regression if any, report path.
@@ -0,0 +1,155 @@
+# Narrow recheck — issue #5 fix round (findings CODE-1, CODE-2)
+
+Repo: clickstream-ch-kafka-superset-demo · branch feature/mentee-path ·
+uncommitted working tree. Review line B (code quality), narrow recheck only.
+Fresh session, no executor history. Sources / git / environment unchanged.
+
+## Method
+
+- Read all working-tree changes vs HEAD in the four generator sources
+  (`startup_history_artifact.py`, `kafka_io.py`, `service.py`,
+  `airflow_control.py`, `state.py`) plus the four changed test files.
+- Traced every producer path for the manifest and the exact ID sets
+  (import, backfill, next-day) and grepped the whole repo for any *reader*
+  of the new chunk topic.
+- Ran focused pytest in an ephemeral uv env (kafka-python 2.0.6, py3.11):
+  the four changed test files → **108 passed**; full generator suite →
+  **216 passed in 13.42 s** (matches the report's "216 passed" claim).
+
+---
+
+## CODE-1 (was High, == TASK-1): manifest payload ceiling — RESOLVED
+
+The single-message ceiling is genuinely gone, and the required incremental
+properties hold. Detail against each verify point:
+
+1. **No path serializes the full ID history into one message.** Exact
+   `click_id`/`user_domain_id` values are produced only inside
+   `_extend_id_set_chain` (`startup_history_artifact.py:347-392`), which
+   slices new IDs into chunks of ≤ `ID_SET_CHUNK_SIZE` (10 000) and emits
+   one Kafka message per chunk via `save_counter_chunk`
+   (`kafka_io.py:245-268`). The manifest's `cumulative_manifest_counters`
+   is `to_state()` — it carries only the chain *summary* (`latest_sha256`,
+   `chunks`, `click_ids`/`user_ids` counts), never the exact values. The
+   segmentation test asserts `"click-1" not in json.dumps(state)`
+   (`test_startup_history_artifact.py`), confirming no exact IDs leak into
+   the manifest record. All three writer paths (import
+   `startup_history_artifact.py:769-773`, backfill `service.py:485-490`,
+   next-day `service.py:659-664`) use the chunked producer. Each chunk is
+   individually bounded (≤10 000 IDs → largest measured 282 KB, well under
+   guard); the manifest itself is ~1.2 KB cumulative + a short `boundaries`
+   list. The day-2 `MessageSizeTooLargeError` that the original finding
+   described can no longer occur.
+
+3. **next-day stays O(new day).** `from_state` restores only the chain
+   *summary* into `_previous_id_set_chain`; `to_state_with_chunks` iterates
+   solely over `self._new_click_ids/_new_user_ids` (IDs first seen this
+   day). No historical chunk is scanned or rewritten; new chunks continue
+   the `sequence`/`previous_sha256` links from the carried summary
+   (`service.py:645-664`). Verified by the segmentation test's continuation
+   assertion (`next_chunks[0].previous_sha256 == chunks[-1][0]`,
+   `chunks == 3`).
+
+4. **Exactness preserved.** The `check` path
+   (`compare_clickhouse_stats_to_manifest`) still compares ClickHouse
+   uniqExact against the manifest `totals` (`events/visits/users/min/max`),
+   which come from the incrementally-maintained counters. The required
+   equality test `test_incremental_counters_equal_full_recompute` genuinely
+   compares incremental (day1 → `to_state` → `from_state` → day2) against a
+   single full recompute of `day1+day2`, on both `to_manifest_topics()` and
+   `to_manifest_totals()`, with `click-1` deliberately re-appearing on day 2
+   to exercise cross-boundary dedup. It was **not** weakened to pass. The
+   two-day service test independently asserts day-2 `topics` **and**
+   `totals` equal a full recompute over `old_batch + published`.
+
+5. **900 000-byte guard is real and explicit.** `_assert_message_size`
+   (`kafka_io.py:270-277`) runs *before send* on every `save` and
+   `save_counter_chunk` call and raises a Russian ValueError naming the
+   byte size and the limit. `save_counter_chunk` additionally recomputes the
+   chunk's own SHA-256 and rejects a mismatched address before send
+   (`kafka_io.py:247-259`). Both failure modes are covered by
+   `test_counter_chunk_has_explicit_kafka_size_and_hash_guards`.
+
+6. **Boundary-ID sourcing is deterministic and correct.**
+   `known_counter_ids_from_state` (`startup_history_artifact.py:295-304`)
+   seeds the resume dedup set from generator state: `click_id`s of
+   `active_visits`, plus `user_domain_id`s of population users with an
+   active or a finished visit — pre-created but never-visited users are
+   excluded. The only prior-day IDs that re-appear in new-day events are
+   active-visit click_ids (seeded → not double-counted) and returning-user
+   ids (seeded via `last_finished_at` → not double-counted); a first-ever
+   visit for a pre-created user is correctly counted as new. This is
+   confirmed empirically by the two-day service test's equality-with-full-
+   recompute assertion, which would fail on any double-count.
+
+### Observation on verify point 2 (non-blocking)
+
+Point 2 asked me to confirm the SHA-256 chunk chain is "validated on read"
+and that "a broken/missing chunk fails loudly." **It is not — because the
+chunks are never read.** A whole-repo grep (`.py/.sh/.yml/.sql/Makefile`)
+finds no consumer of `COUNTER_TOPIC`: there is no `load_counter_chunk`, no
+consumer subscribing to `generator_startup_history_counter_chunks`, and no
+code that re-links or re-hashes the stored chain. The chain's integrity is
+validated only on *write* (each chunk's own hash + the pre-send size guard),
+and the manifest's chain *summary* is checked for internal consistency on
+next-day (`_validate_id_set_chain`, counts must equal totals; head present
+iff `chunks>0`) — but never cross-checked against the actual stored chunk
+messages.
+
+Why this is non-blocking: the executor's report oversells the chain as an
+on-read integrity mechanism, but nothing depends on reading the chunks.
+Exactness of the counts flows entirely through the incremental counters and
+the generator-state-seeded dedup (both verified above); a lost/old local
+state is required to re-run `import`, which rebuilds everything from the
+artifact. So a corrupt or missing chunk causes no wrong result — the chunk
+topic is effectively write-only durable ballast. It grows ~linearly
+(~282 KB/model-day; report projects ~98 MB/year), which for this bounded
+learning stand is tolerable but is dead weight: the chunks are redundant
+with the artifact + generator state. **Recommendation (not blocking): either
+consume the chunks (e.g. to reconstruct/verify exact sets on resume) or drop
+the topic; and correct the report's "validated on read / fails loudly"
+wording, which does not match the code.** This is calibrated the same way
+the original review filed `KafkaDataTopicReader` dead code — a cleanup
+concern, not a correctness defect. It does not reintroduce the ceiling, does
+not corrupt data, and does not weaken exactness, so CODE-1's actual defect
+is resolved.
+
+---
+
+## CODE-2 (was Low): write ordering — RESOLVED
+
+Order is now **chunks → manifest → state** on all three paths:
+
+- import: `startup_history_artifact.py:769-776`
+- backfill: `service.py:485-494`
+- next-day: `service.py:659-666`
+
+The two-day service test asserts the exact sequence
+`["data", "counter_chunk", "manifest", "state"]` for **both** days
+(`test_service.py`), and the backfill flush-failure test asserts
+`save_counter_chunk` was called while `save` (manifest) was **not** and
+state was neither saved nor flushed — locking in that a manifest/state write
+cannot precede the chunks. A manifest failure therefore can no longer leave
+state pointing at an unpublished manifest; the half-advanced window is
+narrowed as intended.
+
+---
+
+## Regressions introduced by the fix
+
+None that affect correctness. One non-blocking cleanup item: the
+`generator_startup_history_counter_chunks` topic is write-only (see CODE-1
+observation) — dead, linearly-growing data with an unexercised on-read
+integrity claim. Low severity, `kafka_io.py:212/245-268`.
+
+---
+
+## VERDICT: APPROVED
+
+Both findings are genuinely resolved: CODE-1's day-2 message ceiling is gone
+(bounded manifest + per-chunk-bounded exact sets, next-day O(new day),
+exactness preserved and honestly tested), and CODE-2's ordering is now
+chunks→manifest→state on all three paths with test assertions. The only new
+wart — a write-only chunk topic whose SHA-256 chain is never read back — is
+non-corrupting cleanup, calibrated as a non-blocking consideration, not a
+defect. Focused and full generator suites green (108 / 216 passed).
@@ -0,0 +1,129 @@
+# Issue 5 report: incremental cumulative manifest counters
+
+## Outcome
+
+Implemented the mandate's preferred seeding design. `backfill` seeds counters while generated batches already flow through `StartupHistoryArtifactBuilder`; `import` seeds them from the artifact topics that are already loaded and validated before Kafka publication. `next-day` restores the cumulative state and adds only the newly generated batches. The full-history `KafkaDataTopicReader.load()` call and its construction were removed from the `next-day` service path.
+
+No git commit was created, and `data/startup_history/reference-world.json.xz` was not changed.
+
+## Seeding and storage design
+
+- The reference artifact remains version 1.0 and contains no cumulative section. `StartupHistoryArtifactBuilder.to_artifact()` deliberately removes the local cumulative sections from its state and manifest copies.
+- On import, the existing artifact checksum is still accepted and validated. The importer builds the new cumulative state from the already loaded topic events, then writes the enriched local state and manifest to Kafka.
+- On backfill, the existing builder counters are persisted directly after successful publication.
+- The local manifest stores:
+  - per-topic row totals and timestamp bounds;
+  - exact `click_id` and `user_domain_id` sets, encoded as sorted JSON, zlib-compressed, and base64-encoded;
+  - the rolling checksum state.
+- The local generator state stores a compact SHA-256 reference to the full cumulative section in the manifest. This prevents mismatched state/manifest pairs without duplicating the large exact ID sets in both Kafka compact-topic messages.
+- The split was chosen after checking the real reference manifest: 26,083 visits and 4,056 users. Duplicating both open UUID arrays in state and manifest would risk Kafka's normal message-size limit. The full exact sets still exist in the manifest, while state proves which manifest counter state it belongs to.
+- Airflow precheck and the generator both validate the state reference, cumulative payload, visible topic counters, and totals before generating. An old local state fails in Russian with an instruction to re-run `import`; there is no fallback to a Kafka history reread.
+
+## Rolling checksum and determinism
+
+Each event is serialized with the existing canonical JSON settings (`sort_keys=True`, `ensure_ascii=True`) and hashed with SHA-256. A topic's rolling checksum is the sum of those 256-bit digests modulo 2^256. This operation is associative and commutative, so restoring the saved accumulator and adding a new day gives the same result as recomputing all events in one pass, independent of batch boundaries. Generation and PRNG usage remain single-threaded and unchanged.
+
+The required equality unit test is `test_incremental_counters_equal_full_recompute`. It seeds the first fixture day, serializes/restores the cumulative state, adds a second day, and compares every topic statistic and every total with a full recompute over the combined fixture. A separate compatibility test proves that an artifact carrying the previous concatenated SHA-256 checksum remains valid and does not need regeneration.
+
+## Verification
+
+### `make test`
+
+Exit code: `0`
+
+Summary:
+
+- Generator image build: successful.
+- Generator tests: `214 passed in 16.00s`.
+- Repository contract tests: `31 passed in 0.26s`.
+- `docker compose config --quiet`: successful.
+
+### `make lint`
+
+Exit code: `0`
+
+Summary:
+
+- Generator image build: successful.
+- Python `compileall`: successful.
+- Bash syntax checks: successful.
+- `docker compose config --quiet`: successful.
+- `git diff --check`: successful.
+
+### `make generated-history-chain-check`
+
+Attempted, exit code: `2`.
+
+The live check was not runnable in this execution context. The script failed while connecting to the Docker API:
+
+```text
+permission denied while trying to connect to the docker API at unix:///var/run/docker.sock
+Ошибка: manifest boundaries должен содержать T0 и T_end
+make: *** [Makefile:64: generated-history-chain-check] Error 1
+```
+
+The second message follows from the missing Docker result rather than a completed chain check. The coordinator should re-run this command in the live acceptance environment.
+
+## Files changed
+
+- `generator/src/clickstream_generator/startup_history_artifact.py`
+- `generator/src/clickstream_generator/state.py`
+- `generator/src/clickstream_generator/service.py`
+- `generator/src/clickstream_generator/airflow_control.py`
+- `generator/tests/test_startup_history_artifact.py`
+- `generator/tests/test_state.py`
+- `generator/tests/test_service.py`
+- `generator/tests/test_airflow_control.py`
+- `docs/runbooks/startup-history.md`
+- `docs/OPERATIONS.md`
+- `docs/ARCHITECTURE.md`
+
+## Open questions and follow-up
+
+- Re-run `make generated-history-chain-check` against a live stand with Docker API access.
+- Live acceptance should compare generation-plus-manifest duration across multiple `next-day` runs. The code path is now O(events in the new day) for generation/counter work, but this sandbox run did not perform the multi-day wall-clock benchmark.
+- ETL remains `full_refresh` exactly as required; its age-dependent cost is still tracked separately.
+
+## Triage round 1
+
+### Findings
+
+- `CODE-1/TASK-1` — FIXED. Exact ID sets are no longer embedded in the single cumulative manifest record. They are split into immutable, content-addressed chunks of at most 10,000 IDs in the separate compact topic `generator_startup_history_counter_chunks`. The main manifest stores cumulative totals, rolling topic checksums, and the SHA-256 head/count of the chunk chain. `next-day` writes only chunks for IDs first seen in the new day; it does not rewrite or scan historical chunks.
+- `CODE-2` — FIXED. `next-day` now flushes new counter chunks, then the main manifest, and saves state last. A manifest failure therefore cannot leave state pointing at an unpublished manifest. Import and backfill use the same chunks -> manifest -> state order.
+
+### Representation and exactness
+
+Raising Kafka message limits alone was rejected because it would preserve the O(history) rewrite and only move the ceiling. Chunking keeps each Kafka value bounded while allowing total stored counter data to grow linearly with world age.
+
+Each chunk contains the exact compressed `click_id` and `user_domain_id` values and the SHA-256 of the previous chunk. The chunk key is its own SHA-256 and is checked before send. The main manifest remains the source of exact `visits` and `users` totals used by the ClickHouse `uniqExact` check. On resume, IDs that can legally reappear across the day boundary come from generator state: active visits, and users that already have an active or completed visit. Unused pre-created population members are deliberately excluded. New visits still receive a new deterministic PRNG UUID. The incremental-vs-full-recompute test compares both topic statistics and complete manifest totals.
+
+Both the main manifest and each chunk have an explicit 900,000-byte pre-send guard with a Russian error. Chunks are in a separate topic so loading the single main manifest key does not scan historical chunk payloads.
+
+The reference artifact in git remains unchanged.
+
+### Measured payload and growth
+
+Measured against `data/startup_history/reference-world.json.xz` (280,437 events, 26,083 visits, 4,056 users):
+
+- main manifest JSON: 3,152 bytes;
+- compact cumulative counter section inside it: 1,219 bytes;
+- exact ID chain: 4 chunks, 846,718 bytes total;
+- largest chunk: 282,324 bytes, 31% of the explicit 900,000-byte guard.
+
+At the reference world's observed average, exact-set storage grows by about 282,239 bytes per model day. Projection: about 28,223,933 bytes (26.9 MiB) at 100 days and 103,017,357 bytes (98.2 MiB) at 365 days, spread across bounded messages. The main manifest grows only through the short `boundaries` list: measured synthetic sizes were 6,023 bytes at 100 days and 13,708 bytes at 365 days.
+
+### Tests changed
+
+- Added a hash-chain segmentation test with a deliberately tiny chunk limit, including continuation from a saved chain and exact duplicate handling at a day boundary.
+- Added explicit oversized-message and wrong-content-hash rejection tests.
+- Updated the required incremental-vs-full-recompute test to provide the exact boundary IDs available from generator state.
+- Updated import tests to assert chunk persistence and the manifest chain head.
+- Updated the two-day service test to assert chunks -> manifest -> state ordering and full manifest totals, including users that were pre-created but had not appeared before the boundary.
+- Existing mismatch, old-state, rolling-checksum, artifact-compatibility, publish-failure, and no-Kafka-reread coverage remains green.
+
+### Verification after fixes
+
+- `make test`: exit code `0`; generator tests `216 passed in 16.46s`, repository contract tests `31 passed in 0.27s`, compose validation successful.
+- `make lint`: exit code `0`; image build, Python compileall, Bash syntax, compose validation, and `git diff --check` successful.
+- `make generated-history-chain-check`: exit code `2`; still not runnable in this execution context because the script cannot access `/var/run/docker.sock`. It then reports missing manifest boundaries as a consequence. The live acceptance environment must rerun it.
+- Working tree remains uncommitted; no git commit was created.
@@ -0,0 +1,43 @@
+# Review focus A: task conformance (issue #5)
+
+You are an independent reviewer, fresh session, no executor history. Do NOT
+modify sources, git state, or permanent environment. You may read anything
+and run focused tests.
+
+Repo: /home/dementev/sources/clickstream-ch-kafka-superset-demo, branch
+feature/mentee-path. Review scope: ALL uncommitted working-tree changes
+(git status/diff vs HEAD).
+
+Mandate the executor worked from:
+/tmp/claude-1000/-home-dementev-sources-clickstream-ch-kafka-superset-demo/0836e970-ee07-4f86-a61f-291f32da525a/scratchpad/mandate-issue5.md
+Also: docs/specs/2026-07-19-mentee-path-redesign.md («Решения» item 4,
+«Чего здесь не делаем»), .scratch/hitl-findings.md finding F9.
+
+Verify CONFORMANCE, not style:
+1. Full Kafka history reread is truly GONE from the next-day path (F9's
+   culprit removed, not bypassed conditionally).
+2. Seeding follows the mandated direction: import seeds from already-read
+   artifact topics; backfill seeds from the generated stream; no lazy
+   full-reread fallback anywhere; old local state fails with a clear
+   Russian error telling to re-run import.
+3. Boundaries: generation not parallelized (single PRNG stream untouched);
+   no incremental ETL; retention untouched; reference artifact file in git
+   unchanged (git diff must not touch data/); docs/course/ untouched.
+4. Issue #5 acceptance criteria coverage: cumulative counters (sums,
+   uid/click_id sets, rolling checksum) in state/manifest; day N ~ day 1
+   plausible from the code path (O(new day events)); required equality test
+   exists and genuinely compares incremental vs full recompute; `check`
+   operation semantics preserved.
+5. Docs updated where state/manifest semantics are user-visible.
+
+Findings: stable IDs TASK-N; each with severity, file:line, violated
+criterion, failure scenario + evidence, minimal way to verify.
+Non-blocking considerations: no ID, no severity.
+VERDICT: APPROVED only if no finding requires a code/docs change; bare
+APPROVED without listing reviewed scope and performed checks does not count.
+
+Write the full report (English) to:
+/tmp/claude-1000/-home-dementev-sources-clickstream-ch-kafka-superset-demo/0836e970-ee07-4f86-a61f-291f32da525a/scratchpad/review-issue5-task.md
+
+Final message: VERDICT, severity counters, one line per finding, one line
+per significant non-blocking consideration, report path.
@@ -0,0 +1,80 @@
+# Code-correctness review — issue #5 (incremental cumulative manifest counters)
+
+Repo: clickstream-ch-kafka-superset-demo · branch feature/mentee-path · uncommitted working tree.
+Reviewer: independent, fresh session. Focus: code correctness and cross-module invariants.
+
+## Reviewed scope
+
+Working-tree changes vs HEAD:
+
+- `generator/src/clickstream_generator/startup_history_artifact.py` (rolling checksum, `to_state`/`from_state`, id-set encode/decode, legacy-checksum fallback, import seeding, artifact stripping)
+- `generator/src/clickstream_generator/state.py` (`cumulative_manifest_counters` field, to_dict/from_dict)
+- `generator/src/clickstream_generator/service.py` (next-day incremental path, removal of `KafkaDataTopicReader` from next-day, guards)
+- `generator/src/clickstream_generator/airflow_control.py` (`assert_next_day_snapshot` cumulative guards)
+- tests: test_startup_history_artifact.py, test_state.py, test_service.py, test_airflow_control.py
+- docs: ARCHITECTURE.md, OPERATIONS.md, runbooks/startup-history.md
+
+## Checks performed
+
+- Read all changed sources + related generation/runtime/kafka_io to confirm batch composition and Kafka limits.
+- Ran targeted new tests and the full generator suite in an ephemeral uv env
+  (`PYTHONPATH=src`, kafka-python 2.0.6): **214 passed** (matches report claim). Docker-based `make test`/`make lint` not runnable here (no docker socket), consistent with the executor's note.
+- Measured the encoded manifest id-set size for realistic uuid4 populations (script in scratchpad).
+- Loaded and validated the REAL shipped artifact `data/startup_history/reference-world.json.xz` to test old-format compatibility empirically (12.4 s validate, OK).
+
+---
+
+## Findings
+
+### CODE-1 — High — manifest message grows with world age and breaks the Kafka 1 MB limit after ~1 next-day
+File: `startup_history_artifact.py:196-207` (`to_state` embeds `_encode_id_set(self.click_ids/user_ids)`), `:480` (`build_manifest` puts `cumulative_manifest_counters` into the manifest), `:602` (import), consumed by `kafka_io.py:219-237` (manifest producer, default `max_request_size`) and `:365-374` (`ensure_topics`, no `max.message.bytes`).
+
+Violated criterion: acceptance "Generation+manifest task on day N takes ~the same time as day 1" and the issue goal "next-day stops getting more expensive as the world ages" — the age-dependent cost is not removed, only moved from read-time to a hard write-time ceiling.
+
+Failure scenario + evidence:
+- The exact `click_id`/`user_domain_id` sets are stored, per event, inside the single manifest Kafka message; they grow monotonically with world age.
+- The shipped reference world already holds **26,083 visits + 4,056 users spanning 3 model days** (verified: totals `events=280437, visits=26083, users=4056, max_event_timestamp=2026-01-03`).
+- Measured encoded size (zlib level 9 + base64 over uuid4): reference world id-sets ≈ **0.81 MB** already; after ~one next-day (~34.8k visits) ≈ **1.1 MB**; a couple of days later multiple MB. Measurement script: `scratchpad/measure.py`.
+- No size override exists anywhere: producer `max_request_size` (kafka-python default 1 048 576), broker `message.max.bytes`, manifest topic `max.message.bytes`, and consumer `max_partition_fetch_bytes` are all at defaults (grep across `generator/`, `infra/`, compose returned nothing). The manifest producer (`KafkaStartupHistoryManifest.save`) does client-side size validation, so a >1 MB manifest raises `MessageSizeTooLargeError` on `send`.
+- Net: the very first next-day after importing the shipped reference world lands at/over the 1 MB boundary; the second next-day exceeds it and `manifest_manager.save()` fails. The feature therefore breaks on roughly the day it is meant to make cheap, and `make generated-history-chain-check` on day seams would fail once the manifest cannot be persisted.
+
+Minimal way to verify: `uv run python scratchpad/measure.py` (shows 0.81 MB at 26 083 visits, 1.59 MB at ~52 k); `grep -rn 'max_request_size\|message.max.bytes\|max.message.bytes\|max_partition_fetch_bytes' generator infra docker-compose*.yml` → no hits; note the reference totals above.
+
+Fix direction (not prescriptive): raise the four Kafka size limits with real headroom (producer + broker + topic + consumer) and/or stop embedding the full exact sets in a single compact-topic message (chunked/segmented store, or a side artifact). This must be resolved or explicitly accepted as a bounded-world limitation before live acceptance — it is currently a silent day-2 break.
+
+### CODE-2 — Low — next-day persists state before manifest, widening the partial-write window
+File: `service.py:641-644` (state.save → state.flush → manifest.save → manifest.flush), vs import order `startup_history_artifact.py:637-639` (manifest first, then state).
+
+Violated criterion: state/manifest coupling robustness under crash / failed write.
+
+Failure scenario + evidence: the new day's events are published and flushed at `service.py:614` before either compact write. State (which carries only the small `manifest_sha256` reference) is then saved and flushed first; if the manifest write then fails — which CODE-1 makes likely — or the process dies between, state points at the new cumulative reference while the manifest is stale. This is caught fail-safe on the next run: `_run_next_day`/`assert_next_day_snapshot` raise `IncompatibleStateError`/`RuntimeError` because `cumulative_counter_reference(counter_state) != counter_reference` (guards at `service.py:530-556`, `airflow_control.py:176-202`). So there is no silent corruption, but the stand is left half-advanced (new events already in Kafka) and requires a clean re-import; manifest-first ordering would shrink the window. The mismatch path IS exercised by tests (`test_airflow_control.py:152-155`, `test_service.py` next-day guards). Non-corrupting, hence Low.
+
+Minimal way to verify: read `service.py:614-644`; observe events flushed then state-before-manifest; the guard/error path is covered by the added tests.
+
+---
+
+## Non-blocking considerations
+
+- Rolling checksum math is sound: per-event SHA-256 digest summed mod 2^256 is associative and commutative, so restore-then-append equals a single full recompute regardless of batch boundaries or order. Serialization (`json.dumps(sort_keys=True, ensure_ascii=True)`) is byte-identical to the pre-existing/legacy checksum input (confirmed by diff). Min/max are order-independent aggregates; visit counts are set-based. The invariant "incremental == full recompute" genuinely holds. I traced the device/geo timestamp derivation (`ManifestCounters.add_batch`, `runtime.py:487-488` releases a page-view's 4 topic events together) and confirmed per-batch vs single-batch derivation converge to the same topic min/max.
+
+- Old-format compatibility works on the REAL artifact, not just synthetically: the shipped `reference-world.json.xz` stores the **legacy concatenated** SHA-256 (verified `stored == legacy: True`, `stored == new-sum: False`), and `validate_startup_history_artifact` passes via the `_LegacyManifestCounters` fallback in `_validate_manifest_topics`. Import then recomputes new-sum counters and overwrites `manifest["topics"]/totals` + cumulative section before writing to Kafka, so the post-import stand is fully new-format and self-consistent. No false-fail. False-pass is negligible: acceptance requires totals (recomputed from events) to match AND (new-sum OR legacy checksum) to match, so a tamper that preserves all totals and collides a checksum is required.
+
+- Equality test is honest: `test_incremental_counters_equal_full_recompute` compares `to_manifest_topics()` (rows, per-topic checksum, min/max) AND `to_manifest_totals()` (events/visits/users/min/max), round-trips through `to_state`/`from_state`, and its two-day fixture makes `click-1` span both days — exercising the cross-batch device/geo derivation, not just self-contained batches.
+
+- Compatibility test is synthetic: `test_legacy_artifact_checksum_remains_valid` reconstructs the legacy checksum in-test rather than pinning the real fixture. The real file passes (I verified manually), so the risk is covered empirically today, but the suite would not catch a future regression against the shipped artifact. A fast smoke test loading `reference-world.json.xz` and asserting validity would guard it (caveat: ~33 MB xz, ~12 s validate — heavier than repo "small slice" test norms).
+
+- `to_state()` is recomputed twice per next-day and per backfill (once for the state reference hash at `service.py:629`/`469`, once inside `build_manifest` at `:480`), each re-running zlib level 9 over the full ~1 MB id-set. Deterministic in-process so correct, but wasteful; import correctly computes it once and reuses (`startup_history_artifact.py:595`).
+
+- `manifest["topics"]` is duplicated inside `manifest["cumulative_manifest_counters"]["topics"]` — the same per-topic block is stored twice in the message, adding to CODE-1's size pressure.
+
+- `KafkaDataTopicReader` (`kafka_io.py:284`) is now dead code — the class remains but has no callers after the next-day rewrite. Cleanup opportunity.
+
+- Russian error texts for the old-state failure are clear and actionable (e.g. "накопительные счётчики state и manifest не совпадают; повторно запустите import эталонного мира") — they name the fix (re-run import) as required by the mandate.
+
+- `_decode_id_set` is robust against corrupt payloads: catches ValueError/TypeError/binascii.Error/zlib.error, enforces list-of-str, and rejects duplicates, all raising clean Russian ValueErrors surfaced as `IncompatibleStateError`.
+
+---
+
+## VERDICT: CHANGES_REQUESTED
+
+CODE-1 (High) requires a change: the incremental manifest embeds unbounded, world-age-growing exact ID sets in one Kafka message and breaks the default 1 MB producer/broker limit after roughly one next-day on the shipped reference world, silently reintroducing the age-dependent failure the issue set out to remove. CODE-2 (Low) is a fail-safe ordering weakness. The checksum math, incremental-vs-full equality, and real old-artifact compatibility are all correct and verified.
@@ -0,0 +1,11 @@
+[2026-07-22T19:12:10.055Z] Final output
+VERDICT: CHANGES_REQUESTED  
+Severity: MAJOR 1, MINOR 0.
+
+TASK-1 — MAJOR: The manifest rewrites all historical ID sets on every run, so it remains O(history), not O(new day), and will eventually exceed Kafka’s default 1 MiB request limit ([kafka-python 2.0.6 docs](https://kafka-python.readthedocs.io/en/2.0.6/apidoc/KafkaProducer.html)).
+
+Non-blocking: Tests and the live chain check could not run because `uv` and Docker are blocked in this sandbox.
+
+Report not created: `/tmp/claude-1000/-home-dementev-sources-clickstream-ch-kafka-superset-demo/0836e970-ee07-4f86-a61f-291f32da525a/scratchpad/review-issue5-task.md`. The read-only filesystem rejected the write.
+
+I did not modify any tracked files or Git state.
@@ -0,0 +1,53 @@
+# Triage round: issue #5 — both review lines, coordinator header
+
+You are the executor continuing your own issue #5 implementation (working
+tree on branch feature/mentee-path; your original mandate:
+mandate-issue5.md in this same directory, your report: report-issue5.md).
+Do the per-finding triage and fix.
+
+## Coordinator header (agreements / contradictions / boundary notes)
+
+- Both independent lines converge on ONE blocking defect:
+  CODE-1 (line B, code quality) == TASK-1 (line A, task conformance).
+  The cumulative manifest payload carries exact click_id/uid sets:
+  ~0.81 MB already on the reference world, Kafka default 1 MiB message
+  limit is not overridden anywhere → manifest save fails after roughly one
+  more next-day. Size is O(history); the issue's disease moves from
+  read-time to a write-time ceiling. No contradictions between the lines.
+- Line B additionally: CODE-2 (Low) — next-day saves state before manifest
+  (import does the reverse), widening the half-advanced window.
+- Full reports (read them directly):
+  - line B (code): review-issue5-code.md (same directory)
+  - line A (task): review-issue5-task.md (verdict text; sandbox blocked its
+    full file write)
+
+## Coordinator decisions for the fix (boundaries, not design)
+
+- The issue's GOAL is time O(new day), no full Kafka reread, no OOM risk.
+  Linear GROWTH of stored counter state over days is acceptable on a demo
+  stand — a hard silent ceiling is not. Whatever representation you choose,
+  the failure mode must be explicit and far away, and the write path must
+  not break within the demo's realistic horizon (say, world age of
+  hundreds of days).
+- Manifest numbers must stay EXACT (the check against ClickHouse uniqExact
+  is the learning value) — no approximate sketches.
+- Reference artifact in git stays unchanged; determinism invariants from
+  the original mandate stay in force.
+- Prefer the simplest solution that meets the above (учебная ценность:
+  a mentee should be able to read and understand it). If two viable
+  options differ in product trade-offs (e.g. raising Kafka topic limits vs
+  restructuring the payload), pick one, implement it, and justify; return
+  NEEDS_DECISION only if the choice genuinely changes user-visible
+  behaviour or issue acceptance criteria.
+
+## What to return
+
+For each finding ID (CODE-1/TASK-1 as one item is fine, CODE-2): one status
+line `ID | FIXED/REJECTED/OUT_OF_SCOPE/NEEDS_DECISION | essence + evidence`.
+Details go to the report: APPEND a "## Triage round 1" section to
+report-issue5.md (same directory): chosen representation and why, measured
+payload size on the reference world after the fix, projection of growth,
+new/changed tests, make test / make lint exit codes.
+
+Rules unchanged: no commits, no changes to data/ artifact, docs updated in
+the same change if state/manifest semantics shift.