fix(migrations): make SQLite-autogenerated revisions portable to Postgres (#342)

.github/workflows/pr.yml

@@ -159,6 +159,37 @@ jobs:
       - name: Module asset build guards
         run: uv run pytest framework/cli/tests/test_module_css_build.py
 
+  # Migrations are autogenerated on SQLite, which hides Postgres-only defects
+  # (integer defaults on boolean columns, undetected expression indexes, enum
+  # types surviving a downgrade). Only a real Postgres run catches them: apply
+  # everything, take it all back down, re-apply, then require zero drift. GH #342.
+  migrations-roundtrip-pg:
+    name: Migrations round trip (Postgres)
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: sm_migrations_check
+        ports: ["5432:5432"]
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 10
+    env:
+      SM_MIGRATIONS_PG_URL: postgresql+asyncpg://postgres:postgres@localhost:5432/sm_migrations_check
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+          cache-dependency-glob: ${{ env.UV_CACHE_GLOB }}
+      - run: make install-py
+      - run: make migrations-roundtrip-pg
+
   e2e-smoke:
     name: E2E smoke (Playwright)
     runs-on: ubuntu-latest
@@ -379,6 +410,7 @@ jobs:
       - js-typecheck
       - js-tests
       - js-build
+      - migrations-roundtrip-pg
       - e2e-smoke
       - perf-guards
       - file-size-check

CLAUDE.md

@@ -39,7 +39,8 @@ All day-to-day tasks go through `make`:
 | `make test-e2e` | Playwright smoke tests (requires `make dev` running + `uv run playwright install chromium`) |
 | `make lint` | Ruff format-check + Ruff + `ty` + Biome + per-workspace `tsc` + 300-line file cap |
 | `make doctor` | Module diagnostics (orphan pages, coupling violations, migration drift, locale checks) — same checks run at prod boot |
-| `make migrate` / `make migration msg="..."` | Apply / autogenerate Alembic migrations |
+| `make migrate` / `make migration msg="..."` | Apply / autogenerate Alembic migrations (autogenerated on SQLite? review per `docs/module-authoring.md` § Migrations) |
+| `make migrations-roundtrip-pg` | Postgres portability gate: `upgrade heads` → `downgrade base` → `upgrade heads` → `alembic check` on `SM_MIGRATIONS_PG_URL` (a disposable DB; CI runs it) |
 | `make new-module name=<name>` | Scaffold a new module package end-to-end |
 | `make gen-pages` | Regenerate `host/client_app/modules.{manifest.json,generated.ts,generated.css}` from installed modules |
 
@@ -112,7 +113,7 @@ Standard mixins in `simple_module_db.mixins`: `AuditMixin`, `SoftDeleteMixin` (b
 
 ## Diagnostic codes
 
-Meaningful codes when reading `make doctor` output: `SM001` missing meta (error), `SM003` orphan page / `SM004` phantom render (warn), `SM007` module overrides no hooks (info), `SM008` duplicate name (error), `SM009` framework→plugin import (error), `SM010` DB revision behind head (error), `SM011` module table not in migration history (warn), `SM012` `register_settings` overridden but nothing on `app.state.<module>` (warn, fires at dev boot only), `SM013`–`SM016` locale issues, `SM017` module ships `.tsx` pages but is missing `package.json`/`tsconfig.json` (warn), `SM018` Inertia `router.{post,patch,put,delete}()` in a page targets a JSON `/api/*` endpoint (warn — Inertia rejects non-Inertia responses), `SM019` module registers view routes (non-empty `view_prefix` + overrides `register_routes`) but overrides neither `register_menu_items` nor `register_permissions` (warn — pages exist with no sidebar entry and no role-editor visibility; admins can't reach them through the UI). Modules whose views are sub-pages of another module typically register permissions to stay discoverable in the role editor without needing their own sidebar entry. `SM020` multiple auth provider modules installed (error), `SM021` no auth provider module installed (warn), `SM022` `@theme`/`@custom-variant`/`@utility` in a module's `styles.css`, where `layer(components)` makes them inert (warn), `SM023` an unlayered rule in a module's `theme.css`, which outranks every Tailwind utility (warn). `SM024` a unique key on a `MultiTenantMixin` table that omits `tenant_id` (warn). `SM025` `multi_tenant` is on but no module registered `app.state.tenant_resolver` (warn — checked at boot after module registration, in every environment, not by the `make doctor` CLI). In production, errors fail boot.
+Meaningful codes when reading `make doctor` output: `SM001` missing meta (error), `SM003` orphan page / `SM004` phantom render (warn), `SM007` module overrides no hooks (info), `SM008` duplicate name (error), `SM009` framework→plugin import (error), `SM010` DB revision behind head (error), `SM011` module table not in migration history (warn), `SM012` `register_settings` overridden but nothing on `app.state.<module>` (warn, fires at dev boot only), `SM013`–`SM016` locale issues, `SM017` module ships `.tsx` pages but is missing `package.json`/`tsconfig.json` (warn), `SM018` Inertia `router.{post,patch,put,delete}()` in a page targets a JSON `/api/*` endpoint (warn — Inertia rejects non-Inertia responses), `SM019` module registers view routes (non-empty `view_prefix` + overrides `register_routes`) but overrides neither `register_menu_items` nor `register_permissions` (warn — pages exist with no sidebar entry and no role-editor visibility; admins can't reach them through the UI). Modules whose views are sub-pages of another module typically register permissions to stay discoverable in the role editor without needing their own sidebar entry. `SM020` multiple auth provider modules installed (error), `SM021` no auth provider module installed (warn), `SM022` `@theme`/`@custom-variant`/`@utility` in a module's `styles.css`, where `layer(components)` makes them inert (warn), `SM023` an unlayered rule in a module's `theme.css`, which outranks every Tailwind utility (warn). `SM024` a unique key on a `MultiTenantMixin` table that omits `tenant_id` (warn). `SM025` `multi_tenant` is on but no module registered `app.state.tenant_resolver` (warn — checked at boot after module registration, in every environment, not by the `make doctor` CLI). `SM026` the database is SQLite and a model declares an expression index it cannot verify (info — autogenerate and `alembic check` skip those on SQLite; add them by hand and run `make migrations-roundtrip-pg`). In production, errors fail boot.
 
 ## Tests & fixtures
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: install install-py install-js dev dev-api dev-ui build test test-py test-py-pg test-js test-e2e bench memray-run memray-flamegraph loadtest loadtest-seed loadtest-memray bench-nav lint doctor migrate migration downgrade migration-history docker-up docker-down kill new-module gen-pages gen-i18n docker-build docker-app docker-compose-app sync-module-deps ci-python-lint ci-python-typecheck ci-js-lint ci-js-typecheck ci-check-file-size ci-check-hardcoded-strings ci-check-untranslated ci-build-packages worker beat worker-docker
+.PHONY: install install-py install-js dev dev-api dev-ui build test test-py test-py-pg test-js test-e2e bench memray-run memray-flamegraph loadtest loadtest-seed loadtest-memray bench-nav lint doctor migrate migration downgrade migration-history migrations-roundtrip-pg docker-up docker-down kill new-module gen-pages gen-i18n docker-build docker-app docker-compose-app sync-module-deps ci-python-lint ci-python-typecheck ci-js-lint ci-js-typecheck ci-check-file-size ci-check-hardcoded-strings ci-check-untranslated ci-build-packages worker beat worker-docker
 
 # Install
 install:
@@ -183,6 +183,18 @@ migration:                  ## Create new migration (usage: make migration msg="
 downgrade:                  ## Downgrade one revision
 	uv run --project host alembic -c host/alembic.ini downgrade -1
 
+# Portability gate for GH #342: revisions are autogenerated on SQLite, which hides
+# Boolean-default, expression-index and enum-downgrade defects until Postgres runs
+# them. Runs on SM_MIGRATIONS_PG_URL, a DISPOSABLE, already-created database
+# (`createdb sm_migrations_check` on the shared dev Postgres); it ends empty.
+SM_MIGRATIONS_PG_URL ?= postgresql+asyncpg://postgres:postgres@localhost:5432/sm_migrations_check
+ALEMBIC_PG = SM_DATABASE_URL=$(SM_MIGRATIONS_PG_URL) uv run --project host alembic -c host/alembic.ini
+migrations-roundtrip-pg:    ## upgrade heads -> downgrade base -> upgrade heads -> alembic check, on Postgres
+	$(ALEMBIC_PG) upgrade heads
+	$(ALEMBIC_PG) downgrade base
+	$(ALEMBIC_PG) upgrade heads
+	$(ALEMBIC_PG) check
+
 migration-history:          ## Show migration history
 	uv run --project host alembic -c host/alembic.ini history --verbose
 

docs/module-authoring.md

@@ -302,6 +302,59 @@ that allowlists only tables owned by installed modules. Any host-defined
 table (e.g. a user table the host dev added directly) is preserved
 untouched by autogenerate.
 
+### Reviewing a revision autogenerated on SQLite
+
+`make migration` runs against whatever `SM_DATABASE_URL` points at, which is
+SQLite by default. SQLite is forgiving, so a revision generated there can fail
+on PostgreSQL while `alembic check` and the boot check report clean. Review each
+generated file against this checklist before treating it as portable
+([GH #342](https://github.com/antosubash/simple_module_python/issues/342)):
+
+1. **Boolean defaults.** A Boolean `server_default` must be `sa.false()` /
+   `sa.true()`, never `sa.text('0')` / `sa.text('1')` — PostgreSQL rejects an
+   integer default on a boolean column (`DatatypeMismatch`). The framework's
+   `render_item` and `process_revision_directives` hooks (already wired in the
+   scaffolded `env.py`) rewrite these for you; check any revision written
+   without them, and always write `server_default=sa.false()` on the model.
+2. **Expression indexes.** SQLite cannot reflect `Index(..., text("lower(email)"))`,
+   so autogenerate skips it and `alembic check` cannot see it. Autogenerate on
+   SQLite logs a banner naming every expression index in your models, and
+   `make doctor` reports `SM026`. Add each one to a revision by hand
+   (`op.create_index(..., [sa.text("lower(email)")], if_not_exists=True)`) and
+   confirm with the PostgreSQL round trip below. The warning is deliberately not
+   an error: it fires on every autogenerate, including unrelated revisions, and
+   the Postgres round trip is the actual gate.
+3. **Enum types on downgrade.** `op.drop_table` leaves the PostgreSQL
+   `CREATE TYPE` behind, so `downgrade` then `upgrade` fails with
+   `DuplicateObject`. Autogenerate never emits the drop. Finish `downgrade()`
+   with the helper, which is a no-op off PostgreSQL:
+
+   ```python
+   from simple_module_db import drop_enums_if_postgres
+
+
+   def downgrade() -> None:
+       op.drop_table("orders_order")
+       drop_enums_if_postgres(op, "orders_status")
+   ```
+
+4. **`SAEnum` server defaults use the member name.** For
+   `sa.Enum(Status)` the stored label is the member *name*
+   (`server_default="PUBLISH"`), not its value (`"publish"`). SQLite accepts
+   either; PostgreSQL rejects the value with `invalid input value for enum`.
+   (A `StrEnum` rendered with `values_callable`, which is what `render_item`
+   emits, stores the *value* — match the default to whichever the column uses.)
+
+Prove it on PostgreSQL before merging. Create a disposable database once
+(`createdb sm_migrations_check` on the shared dev Postgres), then:
+
+```bash
+make migrations-roundtrip-pg   # upgrade heads, downgrade base, upgrade heads, alembic check
+```
+
+CI runs the same target on a Postgres service (`migrations-roundtrip-pg` job in
+`pr.yml`), so a revision that only works on SQLite fails the PR.
+
 ### Multi-module branches
 
 Each module's first revision should set a `branch_labels` tuple matching

docs/reference/diagnostic-codes.md

@@ -34,6 +34,7 @@ The framework runs a set of static checks over installed modules at app boot. Th
 | `SM023` | WARNING | A module's `theme.css` contains an unlayered plain rule (anything but an at-rule or a `:root`-style selector). Unlayered CSS outranks every Tailwind utility. | Move the rule to the module's `styles.css`, which is imported into `layer(components)` so utilities still win. |
 | `SM024` | WARNING | A unique column, constraint or index on a `MultiTenantMixin` table does not include `tenant_id`, so the first tenant to claim a value locks every other tenant out of it. | Make the key per tenant: add `tenant_id` to it (`Index(..., "tenant_id", "slug", unique=True)`). |
 | `SM025` | WARNING | `multi_tenant` is on but no module registered `app.state.tenant_resolver`, so only the principal's `tenant_id` claim can bind a tenant — with most auth providers every tenant-scoped query then fails closed. Checked at boot (it needs the built app), in every environment, not by `make doctor`. | Install the `tenants` module, or register your own `async (Request) -> str \| None` resolver on `app.state.tenant_resolver`. |
+| `SM026` | INFO | The configured database is SQLite and a module model declares an expression index (e.g. `lower(email)`). SQLite cannot reflect those, so autogenerate and `alembic check` cannot verify them and a clean diff says nothing about them. | Add the index to a revision by hand and verify with `make migrations-roundtrip-pg` (PostgreSQL). See [Migrations](/module-authoring#reviewing-a-revision-autogenerated-on-sqlite). |
 
 `SM022`/`SM023` are the two halves of the same invariant: a module's optional [`theme.css` is imported unlayered and `styles.css` into `layer(components)`](/module-authoring#styling), and CSS put in the wrong one silently does nothing (or silently wins everything).
 

framework/cli/simple_module_cli/templates/host/migrations/env.py

@@ -31,8 +31,9 @@
 
 target_metadata = build_module_metadata()
 include_object = make_include_object(target_metadata)
-# Re-emit expression-based indexes (e.g. ``lower(email)``) that autogenerate
-# silently drops under SQLite. See ``make_process_revision_directives`` docstring.
+# Re-emit expression-based indexes that autogenerate drops under SQLite, rewrite
+# Boolean 0/1 server defaults to sa.false()/sa.true(), and warn that SQLite cannot
+# verify expression indexes (GH #342) — see make_process_revision_directives.
 process_revision_directives = make_process_revision_directives(target_metadata)
 
 

framework/core/simple_module_core/__main__.py

@@ -73,6 +73,16 @@ def _load_i18n_settings_from_env() -> tuple[list[str] | None, str]:
     return supported, default
 
 
+def _database_dialect() -> str:
+    """Dialect name of ``SM_DATABASE_URL`` (``.env`` merged by the i18n loader first).
+
+    Defaults to ``sqlite`` like ``Settings.database_url``. Reads the env var
+    directly for the same reason the i18n settings do: no hosting import.
+    """
+    url = os.environ.get("SM_DATABASE_URL", "sqlite+aiosqlite:///./app.db")
+    return url.split(":", 1)[0].split("+", 1)[0]
+
+
 def _discover_extra_locale_sources() -> list[tuple[str, str, Path]]:
     """Return ``[(reporter, namespace, path), ...]`` for host + ui locale dirs."""
     # Anchor on the same project root the `.env` was loaded from
@@ -135,6 +145,7 @@ def main() -> int:
         i18n_supported_locales=supported,
         i18n_default_locale=default,
         i18n_extra_sources=extra,
+        database_dialect=_database_dialect(),
     )
     print_diagnostics(diagnostics)
 

framework/core/simple_module_core/diagnostics/_expression_index.py

@@ -0,0 +1,106 @@
+"""Expression-index diagnostics (SM026).
+
+SQLAlchemy cannot reflect expression indexes (``lower(email)``) on SQLite, so
+autogenerate and ``alembic check`` silently skip them there and report a clean
+diff. That is "unverified", not "clean": on Postgres the same index is either
+missing from the history or drifted. SM026 says so whenever the configured
+database is SQLite and any module model declares such an index.
+
+Duck-typed on SQLAlchemy ``Table``/``Index`` objects (core does not depend on
+SQLAlchemy).
+"""
+
+from __future__ import annotations
+
+import importlib
+import logging
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Any
+
+from simple_module_core.diagnostics._types import Diagnostic, DiagnosticLevel
+
+if TYPE_CHECKING:
+    from simple_module_core.module import ModuleBase
+
+logger = logging.getLogger(__name__)
+
+
+def index_is_expression_based(index: Any) -> bool:
+    """True when any element of ``index`` is an expression rather than a plain column.
+
+    A plain ``Column`` is attached to its table; a ``text()`` or function
+    element is not.
+    """
+    return any(getattr(expr, "table", None) is None for expr in index.expressions)
+
+
+def find_expression_indexes(tables: Iterable[Any]) -> list[tuple[str, str]]:
+    """Return sorted ``(table name, index name)`` pairs for expression indexes."""
+    found = {
+        (table.name, index.name or "<unnamed>")
+        for table in tables
+        for index in table.indexes
+        if index_is_expression_based(index)
+    }
+    return sorted(found)
+
+
+def module_all_tables(mod: ModuleBase) -> list[Any]:
+    """Every table declared in the module's ``models`` submodule."""
+    pkg = type(mod).__module__.rsplit(".", 1)[0]
+    try:
+        models = importlib.import_module(f"{pkg}.models")
+    except ModuleNotFoundError:
+        return []
+    except Exception:  # pragma: no cover - a broken models module fails elsewhere, loudly
+        logger.debug("Could not import %s.models for SM026", pkg, exc_info=True)
+        return []
+    seen: dict[str, Any] = {}
+    for value in vars(models).values():
+        table = getattr(value, "__table__", None)
+        if (
+            isinstance(value, type)
+            and table is not None
+            and hasattr(table, "indexes")
+            # Skip models re-imported from another module, or each importer would
+            # be told about an index it does not own.
+            and getattr(value, "__module__", "").startswith(pkg)
+        ):
+            seen.setdefault(table.name, table)
+    return list(seen.values())
+
+
+def check_expression_indexes_unverifiable(
+    tables: Iterable[Any], dialect: str | None, module_name: str
+) -> list[Diagnostic]:
+    """SM026: INFO on SQLite when the models declare indexes it cannot verify."""
+    if dialect != "sqlite":
+        return []
+    found = find_expression_indexes(tables)
+    if not found:
+        return []
+    names = ", ".join(f"{idx} ({tbl})" for tbl, idx in found)
+    return [
+        Diagnostic(
+            level=DiagnosticLevel.INFO,
+            code="SM026",
+            message=(
+                f"SQLite cannot reflect expression indexes, so autogenerate and "
+                f"`alembic check` cannot verify {names}; a clean diff here says nothing "
+                f"about them"
+            ),
+            module_name=module_name,
+            suggestion=(
+                "Add them to a revision by hand and verify on Postgres: "
+                "`make migrations-roundtrip-pg`"
+            ),
+        )
+    ]
+
+
+__all__ = [
+    "check_expression_indexes_unverifiable",
+    "find_expression_indexes",
+    "index_is_expression_based",
+    "module_all_tables",
+]