Use OpenMS-Insight tables on the downstream pages (#38 rebased onto main's helper)

CLAUDE.md

@@ -169,5 +169,7 @@ docker-compose up --build
 - Widget keys must match parameter keys in `default-parameters.json`
 - Workflow names use lowercase with hyphens: "My Workflow" -> "my-workflow"
 - Use `show_fig()` and `show_table()` from `src/common/common.py` for consistent display
+- Show DataFrames with `show_insight_table(df, name=...)` from `src/common/insight_tables.py` (or `show_table()`), not `st.dataframe()`: it uses OpenMS-Insight's server-side-paginated `Table`, so only the visible page is sent to the browser. `st.dataframe` serializes the whole frame and is extremely slow on full-scale datasets. Only use it for tiny fixed-size previews (e.g. `.head(10)`)
+- Insight components request a rerun the first time they render, so never draw them only inside `if st.button(...):`. Store the result in `st.session_state` and render it outside the button block
 - Use `@st.fragment` on methods that should partially rerun (configure, results)
 - TOPP tool parameters use colon-separated paths: `"algorithm:section:param_name"`

content/filtering.py

@@ -6,6 +6,7 @@
 import streamlit as st
 from src.common.common import page_setup, save_params, show_fig
 from src.common.postprocessing_plots import filter_threshold_curve
+from src.common.insight_tables import show_insight_table
 from src.common.results_helpers import (
     clear_downstream_steps,
     get_abundance_data,
@@ -68,7 +69,7 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
 st.markdown(
     f"Currently displaying **{pivot_df.shape[0]}** proteins and **{len(sample_cols)}** samples before filtering."
 )
-st.dataframe(pivot_df, use_container_width=True)
+show_insight_table(pivot_df, name="filtering-original")
 
 st.markdown("---")
 
@@ -192,9 +193,13 @@ def run_filter(value: float) -> pl.LazyFrame:
     st.session_state["filtered_df"] = filtered_df
     clear_downstream_steps("filtered_df")
 
-    # Layout response metrics and the filtered matrix
     st.success(f"Successfully applied **{filter_method}** filter!")
 
+# Results are rendered from session state, outside the button block: the Insight
+# table triggers a rerun the first time it is drawn, which would reset the button
+# and wipe anything shown only while it is pressed.
+filtered_df = st.session_state.get("filtered_df")
+if filtered_df is not None:
     # Display dataset scale compression stats
     col1, col2, col3 = st.columns(3)
     col1.metric("Original Proteins", pivot_df.shape[0])
@@ -209,4 +214,4 @@ def run_filter(value: float) -> pl.LazyFrame:
             "The filtered table is empty. Try relaxing the threshold constraints."
         )
     else:
-        st.dataframe(filtered_df, use_container_width=True)
+        show_insight_table(filtered_df, name="filtering-filtered")

content/imputation.py

@@ -6,6 +6,7 @@
 import streamlit as st
 from src.common.common import page_setup, save_params, show_fig
 from src.common.postprocessing_plots import imputed_value_preview, missingness_vs_intensity
+from src.common.insight_tables import show_insight_table
 from src.common.results_helpers import (
     clear_downstream_steps,
     get_abundance_data,
@@ -76,7 +77,7 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
 st.markdown(
     f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples before imputation."
 )
-st.dataframe(base_df, use_container_width=True)
+show_insight_table(base_df, name="imputation-input")
 
 st.markdown("---")
 
@@ -188,6 +189,10 @@ def run_imputation() -> pl.LazyFrame:
 
     st.success(f"Successfully finalized **{impute_category}** imputation step!")
 
-    # Calculate and display a quick performance matrix check
+# Results are rendered from session state, outside the button block: the Insight
+# table triggers a rerun the first time it is drawn, which would reset the button
+# and wipe anything shown only while it is pressed.
+imputed_df = st.session_state.get("imputed_df")
+if imputed_df is not None:
     st.subheader("Imputed Result Table")
-    st.dataframe(imputed_df, use_container_width=True)
+    show_insight_table(imputed_df, name="imputation-result")

content/normalization.py

@@ -6,6 +6,7 @@
 import streamlit as st
 from src.common.common import page_setup, save_params, show_fig
 from src.common.postprocessing_plots import sample_distributions
+from src.common.insight_tables import show_insight_table
 from src.common.results_helpers import (
     clear_downstream_steps,
     get_abundance_data,
@@ -101,7 +102,7 @@ def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None:
 st.markdown(
     f"Currently displaying **{base_df.shape[0]}** rows and **{len(sample_cols)}** samples entering the normalization block."
 )
-st.dataframe(base_df, use_container_width=True)
+show_insight_table(base_df, name="normalization-input")
 
 st.markdown("### Pipeline Overview")
 st.caption("Data flows in order: Filtering -> Imputation -> Normalization")
@@ -283,12 +284,17 @@ def run_normalization() -> pl.LazyFrame:
 
         st.success("Successfully executed all selected normalization pipelines!")
 
-        # Display the finalized transformation matrix view
-        st.subheader("Normalized Abundance Table")
-        st.dataframe(normalized_df, use_container_width=True)
-
     except ValueError as val_err:
         # Gracefully handle validation failures raised from the engine layers (e.g., missing reference protein)
         st.error(f"Engine Configuration Error: {str(val_err)}")
     except Exception as e:
-        st.error(f"An unexpected pipeline error occurred: {str(e)}")
+        st.error(f"An unexpected pipeline error occurred: {str(e)}")
+
+# Results are rendered from session state, outside the button block: the Insight
+# table triggers a rerun the first time it is drawn, which would reset the button
+# and wipe anything shown only while it is pressed.
+normalized_df = st.session_state.get("normalized_df")
+if normalized_df is not None:
+    # Display the finalized transformation matrix view
+    st.subheader("Normalized Abundance Table")
+    show_insight_table(normalized_df, name="normalization-result")

content/results_pathway_analysis.py

@@ -9,6 +9,7 @@
 from scipy.stats import fisher_exact
 from pathlib import Path
 from src.common.common import page_setup
+from src.common.insight_tables import show_insight_table
 from src.common.results_helpers import get_abundance_data
 
 # ================================
@@ -216,7 +217,7 @@ def run_go(go_type):
         st.info("No protein-level data available.")
     else:
         st.session_state["pivot_df"] = pivot_df
-        st.dataframe(pivot_df.sort_values("p-value"), width="stretch")
+        show_insight_table(pivot_df.sort_values("p-value"), name="pathway-protein")
 
 # ======================================================
 # GO Enrichment Results 
@@ -255,4 +256,4 @@ def run_go(go_type):
                 st.plotly_chart(fig, width="stretch")
 
                 st.markdown(f"#### {go_type} Enrichment Results")
-                st.dataframe(df_go, width="stretch")
+                show_insight_table(df_go, name=f"pathway-go-{go_type}", height=350)

content/results_pca.py

@@ -3,6 +3,7 @@
 import polars as pl
 import streamlit as st
 from src.common.common import page_setup
+from src.common.insight_tables import show_insight_table
 from src.common.results_helpers import (
     get_abundance_data,
     get_active_table,
@@ -64,7 +65,7 @@
     f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples "
     f"belonging to **{len(unique_groups)} groups** ({', '.join(unique_groups)})."
 )
-st.dataframe(base_df, use_container_width=True)
+show_insight_table(base_df, name="pca-input")
 
 st.markdown("---")
 

content/results_proteomicslfq.py

@@ -5,6 +5,7 @@
 import plotly.express as px
 
 from src.common.common import page_setup
+from src.common.insight_tables import show_insight_table
 from src.common.results_helpers import get_abundance_data
 
 # ================================
@@ -52,7 +53,7 @@
         st.info("No protein-level data available.")
     else:
         st.session_state["pivot_df"] = pivot_df
-        st.dataframe(pivot_df, use_container_width=True)
+        show_insight_table(pivot_df, name="proteomicslfq-protein")
 
 # ======================================================
 # GO Enrichment Results 
@@ -97,4 +98,4 @@
                 st.plotly_chart(fig, use_container_width=True)
 
                 st.markdown(f"#### {go_type} Enrichment Results")
-                st.dataframe(df_go, use_container_width=True)
+                show_insight_table(df_go, name=f"proteomicslfq-go-{go_type}", height=350)

content/statistical.py

@@ -6,6 +6,7 @@
 import streamlit as st
 from src.common.common import page_setup, save_params, show_fig
 from src.common.postprocessing_plots import pvalue_histogram
+from src.common.insight_tables import show_insight_table
 from src.common.results_helpers import (
     get_abundance_data,
     get_id_column,
@@ -101,7 +102,7 @@
 st.markdown(
     f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples belonging to **{group_count} groups** ({', '.join(unique_groups)})."
 )
-st.dataframe(base_df, use_container_width=True)
+show_insight_table(base_df, name="statistical-input")
 
 st.markdown("---")
 
@@ -184,17 +185,22 @@
         st.session_state["statistics_df"] = statistics_df
 
         st.success(f"Successfully calculated **{selected_method}** test with **{selected_fdr}** FDR correction!")
-
-        # Display the finalized statistics table view
-        st.subheader("Statistical Analysis Results")
-        st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`")
-        st.dataframe(statistics_df, use_container_width=True)
-
+
     except ValueError as val_err:
         st.error(f"Engine Validation Fallure: {str(val_err)}")
     except Exception as e:
         st.error(f"An unexpected pipeline error occurred: {str(e)}")
 
+# Results are rendered from session state, outside the button block: the Insight
+# table triggers a rerun the first time it is drawn, which would reset the button
+# and wipe anything shown only while it is pressed.
+statistics_df = st.session_state.get("statistics_df")
+if statistics_df is not None:
+    # Display the finalized statistics table view
+    st.subheader("Statistical Analysis Results")
+    st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`")
+    show_insight_table(statistics_df, name="statistical-result")
+
 # --- SECTION 4: Test diagnostics ---
 if st.session_state.get("statistics_df") is not None:
     with st.expander("📈 Check the test result", expanded=True):

src/common/common.py

@@ -885,20 +885,23 @@ def get_current_chunk(df, chunk_size, chunk_index):
         return base_index + rows[0]
 
 
-def show_table(df: pd.DataFrame, download_name: str = "") -> None:
+def show_table(df: pd.DataFrame, download_name: str = "", key: str | None = None) -> None:
     """
-    Displays a pandas dataframe using Streamlit's `dataframe` function and
-    provides a download button for the same table.
+    Displays a pandas dataframe with the OpenMS-Insight `Table` component
+    (server-side pagination, see `src.common.insight_tables.show_insight_table`)
+    and provides a download button for the same table.
 
     Args:
         df (pd.DataFrame): The pandas dataframe to display.
         download_name (str): The name to give to the downloaded file. Defaults to empty string.
+        key (str): Unique name of the table. Defaults to `download_name`.
 
     Returns:
-        df (pd.DataFrame): The possibly edited dataframe.
+        df (pd.DataFrame): The dataframe.
     """
-    # Show dataframe using container width
-    st.dataframe(df, use_container_width=True)
+    from src.common.insight_tables import show_insight_table
+
+    show_insight_table(df, name=key or download_name.replace(" ", "-") or "table")
     # Show download button with the given download name for the table if name is given
     if download_name:
         st.download_button(

src/common/insight_tables.py

@@ -6,13 +6,42 @@
 """
 import hashlib
 import shutil
+import tempfile
 from pathlib import Path
 
 import pandas as pd
 import polars as pl
+import streamlit as st
 from openms_insight import Table
 
 ROW_FIELD = "row"
+# Rows sampled when fingerprinting a dataframe that has no source files.
+_FINGERPRINT_SAMPLE_ROWS = 2000
+
+
+def _fingerprint_dataframe(df: pd.DataFrame) -> str:
+    """Cheap content hash of a dataframe, for tables computed in the session.
+
+    Hashing every cell on every Streamlit rerun would defeat the purpose for
+    large tables, so this combines the shape/schema, the column sums and an
+    evenly spaced sample of rows (always including the first and last row).
+    """
+    digest = hashlib.sha256()
+    digest.update(repr((df.shape, [str(c) for c in df.columns], [str(t) for t in df.dtypes])).encode())
+    digest.update(df.select_dtypes(include="number").sum().to_numpy(dtype="float64").tobytes())
+    if len(df):
+        step = max(len(df) // _FINGERPRINT_SAMPLE_ROWS, 1)
+        rows = df.iloc[::step].astype(str)
+        digest.update(pd.util.hash_pandas_object(rows, index=False).values.tobytes())
+        digest.update(df.iloc[[-1]].astype(str).to_csv(index=False).encode())
+    return digest.hexdigest()
+
+
+def _default_cache_dir() -> Path:
+    """Cache location for tables that are not tied to a workflow result directory."""
+    workspace = st.session_state.get("workspace")
+    base = Path(workspace) if workspace else Path(tempfile.gettempdir(), "quantms-web")
+    return base / "insight_tables"
 
 
 def _cache_key(source_files, extra: str) -> str:
@@ -31,7 +60,8 @@ def _to_polars(df: pd.DataFrame) -> tuple[pl.DataFrame, list[dict]]:
     Tabulator reads a "." in a field name as a nested lookup, so such columns get
     a safe field name and keep their original name as the column title.
     """
-    df = df.reset_index(drop=True)
+    # A named index usually holds the IDs, so keep it as a column
+    df = df.reset_index(drop=all(n is None for n in df.index.names))
     renames, column_definitions, used = {}, [], {ROW_FIELD}
     for column in df.columns:
         field = str(column).replace(".", "_")
@@ -55,7 +85,7 @@ def _to_polars(df: pd.DataFrame) -> tuple[pl.DataFrame, list[dict]]:
 def show_insight_table(
     df: pd.DataFrame,
     name: str,
-    cache_dir,
+    cache_dir=None,
     source_files=(),
     extra_key: str = "",
     title: str | None = None,
@@ -67,20 +97,29 @@ def show_insight_table(
 
     The cache is keyed on ``source_files`` (path and mtime) and ``extra_key``;
     when either changes (a rerun, new group assignments) a new cache replaces the
-    old one, so the page never shows a previous run's rows.
+    old one, so the page never shows a previous run's rows. Without
+    ``source_files`` (a table computed in the session, e.g. a downstream
+    analysis step) it is keyed on a fingerprint of the table's content instead.
+
+    Insight components request a rerun the first time they render, so don't
+    draw one only inside ``if st.button(...):``; keep the result in session
+    state and render it outside the button block.
 
     Args:
         df: Table to show.
         name: Stable name for this table; must be unique per page.
-        cache_dir: Directory holding openms-insight caches.
+        cache_dir: Directory holding openms-insight caches. Defaults to
+            ``<workspace>/insight_tables``.
         source_files: Files the table is derived from.
         extra_key: Anything else the table depends on (e.g. parameters).
         title: Title, also used as the CSV download's file name.
         height: Component height in pixels.
         page_size: Rows per page.
         **table_kwargs: Passed on to ``Table`` (e.g. ``initial_sort``).
     """
-    cache_dir = Path(cache_dir)
+    cache_dir = Path(cache_dir) if cache_dir else _default_cache_dir()
+    if not source_files:
+        extra_key = f"{extra_key}|{_fingerprint_dataframe(df)}"
     cache_id = f"{name}_{_cache_key(source_files, extra_key)}"
 
     if not (cache_dir / cache_id).is_dir():