diff --git a/CLAUDE.md b/CLAUDE.md index e0e0a22..320c58b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -169,5 +169,7 @@ docker-compose up --build - Widget keys must match parameter keys in `default-parameters.json` - Workflow names use lowercase with hyphens: "My Workflow" -> "my-workflow" - Use `show_fig()` and `show_table()` from `src/common/common.py` for consistent display +- Show DataFrames with `show_insight_table(df, name=...)` from `src/common/insight_tables.py` (or `show_table()`), not `st.dataframe()`: it uses OpenMS-Insight's server-side-paginated `Table`, so only the visible page is sent to the browser. `st.dataframe` serializes the whole frame and is extremely slow on full-scale datasets. Only use it for tiny fixed-size previews (e.g. `.head(10)`) +- Insight components request a rerun the first time they render, so never draw them only inside `if st.button(...):`. Store the result in `st.session_state` and render it outside the button block - Use `@st.fragment` on methods that should partially rerun (configure, results) - TOPP tool parameters use colon-separated paths: `"algorithm:section:param_name"` diff --git a/content/filtering.py b/content/filtering.py index 3e6694a..b9e3f1a 100644 --- a/content/filtering.py +++ b/content/filtering.py @@ -6,6 +6,7 @@ import streamlit as st from src.common.common import page_setup, save_params, show_fig from src.common.postprocessing_plots import filter_threshold_curve +from src.common.insight_tables import show_insight_table from src.common.results_helpers import ( clear_downstream_steps, get_abundance_data, @@ -68,7 +69,7 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: st.markdown( f"Currently displaying **{pivot_df.shape[0]}** proteins and **{len(sample_cols)}** samples before filtering." ) -st.dataframe(pivot_df, use_container_width=True) +show_insight_table(pivot_df, name="filtering-original") st.markdown("---") @@ -192,9 +193,13 @@ def run_filter(value: float) -> pl.LazyFrame: st.session_state["filtered_df"] = filtered_df clear_downstream_steps("filtered_df") - # Layout response metrics and the filtered matrix st.success(f"Successfully applied **{filter_method}** filter!") +# Results are rendered from session state, outside the button block: the Insight +# table triggers a rerun the first time it is drawn, which would reset the button +# and wipe anything shown only while it is pressed. +filtered_df = st.session_state.get("filtered_df") +if filtered_df is not None: # Display dataset scale compression stats col1, col2, col3 = st.columns(3) col1.metric("Original Proteins", pivot_df.shape[0]) @@ -209,4 +214,4 @@ def run_filter(value: float) -> pl.LazyFrame: "The filtered table is empty. Try relaxing the threshold constraints." ) else: - st.dataframe(filtered_df, use_container_width=True) \ No newline at end of file + show_insight_table(filtered_df, name="filtering-filtered") diff --git a/content/imputation.py b/content/imputation.py index 9d4b802..89084df 100644 --- a/content/imputation.py +++ b/content/imputation.py @@ -6,6 +6,7 @@ import streamlit as st from src.common.common import page_setup, save_params, show_fig from src.common.postprocessing_plots import imputed_value_preview, missingness_vs_intensity +from src.common.insight_tables import show_insight_table from src.common.results_helpers import ( clear_downstream_steps, get_abundance_data, @@ -76,7 +77,7 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: st.markdown( f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples before imputation." ) -st.dataframe(base_df, use_container_width=True) +show_insight_table(base_df, name="imputation-input") st.markdown("---") @@ -188,6 +189,10 @@ def run_imputation() -> pl.LazyFrame: st.success(f"Successfully finalized **{impute_category}** imputation step!") - # Calculate and display a quick performance matrix check +# Results are rendered from session state, outside the button block: the Insight +# table triggers a rerun the first time it is drawn, which would reset the button +# and wipe anything shown only while it is pressed. +imputed_df = st.session_state.get("imputed_df") +if imputed_df is not None: st.subheader("Imputed Result Table") - st.dataframe(imputed_df, use_container_width=True) \ No newline at end of file + show_insight_table(imputed_df, name="imputation-result") diff --git a/content/normalization.py b/content/normalization.py index 1863fd8..8dad683 100644 --- a/content/normalization.py +++ b/content/normalization.py @@ -6,6 +6,7 @@ import streamlit as st from src.common.common import page_setup, save_params, show_fig from src.common.postprocessing_plots import sample_distributions +from src.common.insight_tables import show_insight_table from src.common.results_helpers import ( clear_downstream_steps, get_abundance_data, @@ -101,7 +102,7 @@ def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None: st.markdown( f"Currently displaying **{base_df.shape[0]}** rows and **{len(sample_cols)}** samples entering the normalization block." ) -st.dataframe(base_df, use_container_width=True) +show_insight_table(base_df, name="normalization-input") st.markdown("### Pipeline Overview") st.caption("Data flows in order: Filtering -> Imputation -> Normalization") @@ -283,12 +284,17 @@ def run_normalization() -> pl.LazyFrame: st.success("Successfully executed all selected normalization pipelines!") - # Display the finalized transformation matrix view - st.subheader("Normalized Abundance Table") - st.dataframe(normalized_df, use_container_width=True) - except ValueError as val_err: # Gracefully handle validation failures raised from the engine layers (e.g., missing reference protein) st.error(f"Engine Configuration Error: {str(val_err)}") except Exception as e: - st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file + st.error(f"An unexpected pipeline error occurred: {str(e)}") + +# Results are rendered from session state, outside the button block: the Insight +# table triggers a rerun the first time it is drawn, which would reset the button +# and wipe anything shown only while it is pressed. +normalized_df = st.session_state.get("normalized_df") +if normalized_df is not None: + # Display the finalized transformation matrix view + st.subheader("Normalized Abundance Table") + show_insight_table(normalized_df, name="normalization-result") \ No newline at end of file diff --git a/content/results_pathway_analysis.py b/content/results_pathway_analysis.py index f5eb4c1..d030a44 100644 --- a/content/results_pathway_analysis.py +++ b/content/results_pathway_analysis.py @@ -9,6 +9,7 @@ from scipy.stats import fisher_exact from pathlib import Path from src.common.common import page_setup +from src.common.insight_tables import show_insight_table from src.common.results_helpers import get_abundance_data # ================================ @@ -216,7 +217,7 @@ def run_go(go_type): st.info("No protein-level data available.") else: st.session_state["pivot_df"] = pivot_df - st.dataframe(pivot_df.sort_values("p-value"), width="stretch") + show_insight_table(pivot_df.sort_values("p-value"), name="pathway-protein") # ====================================================== # GO Enrichment Results @@ -255,4 +256,4 @@ def run_go(go_type): st.plotly_chart(fig, width="stretch") st.markdown(f"#### {go_type} Enrichment Results") - st.dataframe(df_go, width="stretch") \ No newline at end of file + show_insight_table(df_go, name=f"pathway-go-{go_type}", height=350) \ No newline at end of file diff --git a/content/results_pca.py b/content/results_pca.py index a7baf06..aa2875d 100644 --- a/content/results_pca.py +++ b/content/results_pca.py @@ -3,6 +3,7 @@ import polars as pl import streamlit as st from src.common.common import page_setup +from src.common.insight_tables import show_insight_table from src.common.results_helpers import ( get_abundance_data, get_active_table, @@ -64,7 +65,7 @@ f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples " f"belonging to **{len(unique_groups)} groups** ({', '.join(unique_groups)})." ) -st.dataframe(base_df, use_container_width=True) +show_insight_table(base_df, name="pca-input") st.markdown("---") diff --git a/content/results_proteomicslfq.py b/content/results_proteomicslfq.py index fde2ab9..ee5830a 100644 --- a/content/results_proteomicslfq.py +++ b/content/results_proteomicslfq.py @@ -5,6 +5,7 @@ import plotly.express as px from src.common.common import page_setup +from src.common.insight_tables import show_insight_table from src.common.results_helpers import get_abundance_data # ================================ @@ -52,7 +53,7 @@ st.info("No protein-level data available.") else: st.session_state["pivot_df"] = pivot_df - st.dataframe(pivot_df, use_container_width=True) + show_insight_table(pivot_df, name="proteomicslfq-protein") # ====================================================== # GO Enrichment Results @@ -97,4 +98,4 @@ st.plotly_chart(fig, use_container_width=True) st.markdown(f"#### {go_type} Enrichment Results") - st.dataframe(df_go, use_container_width=True) \ No newline at end of file + show_insight_table(df_go, name=f"proteomicslfq-go-{go_type}", height=350) \ No newline at end of file diff --git a/content/statistical.py b/content/statistical.py index d8fe7c1..e814e53 100644 --- a/content/statistical.py +++ b/content/statistical.py @@ -6,6 +6,7 @@ import streamlit as st from src.common.common import page_setup, save_params, show_fig from src.common.postprocessing_plots import pvalue_histogram +from src.common.insight_tables import show_insight_table from src.common.results_helpers import ( get_abundance_data, get_id_column, @@ -101,7 +102,7 @@ st.markdown( f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples belonging to **{group_count} groups** ({', '.join(unique_groups)})." ) -st.dataframe(base_df, use_container_width=True) +show_insight_table(base_df, name="statistical-input") st.markdown("---") @@ -184,17 +185,22 @@ st.session_state["statistics_df"] = statistics_df st.success(f"Successfully calculated **{selected_method}** test with **{selected_fdr}** FDR correction!") - - # Display the finalized statistics table view - st.subheader("Statistical Analysis Results") - st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`") - st.dataframe(statistics_df, use_container_width=True) - + except ValueError as val_err: st.error(f"Engine Validation Fallure: {str(val_err)}") except Exception as e: st.error(f"An unexpected pipeline error occurred: {str(e)}") +# Results are rendered from session state, outside the button block: the Insight +# table triggers a rerun the first time it is drawn, which would reset the button +# and wipe anything shown only while it is pressed. +statistics_df = st.session_state.get("statistics_df") +if statistics_df is not None: + # Display the finalized statistics table view + st.subheader("Statistical Analysis Results") + st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`") + show_insight_table(statistics_df, name="statistical-result") + # --- SECTION 4: Test diagnostics --- if st.session_state.get("statistics_df") is not None: with st.expander("📈 Check the test result", expanded=True): diff --git a/src/common/common.py b/src/common/common.py index c048064..e913a28 100644 --- a/src/common/common.py +++ b/src/common/common.py @@ -885,20 +885,23 @@ def get_current_chunk(df, chunk_size, chunk_index): return base_index + rows[0] -def show_table(df: pd.DataFrame, download_name: str = "") -> None: +def show_table(df: pd.DataFrame, download_name: str = "", key: str | None = None) -> None: """ - Displays a pandas dataframe using Streamlit's `dataframe` function and - provides a download button for the same table. + Displays a pandas dataframe with the OpenMS-Insight `Table` component + (server-side pagination, see `src.common.insight_tables.show_insight_table`) + and provides a download button for the same table. Args: df (pd.DataFrame): The pandas dataframe to display. download_name (str): The name to give to the downloaded file. Defaults to empty string. + key (str): Unique name of the table. Defaults to `download_name`. Returns: - df (pd.DataFrame): The possibly edited dataframe. + df (pd.DataFrame): The dataframe. """ - # Show dataframe using container width - st.dataframe(df, use_container_width=True) + from src.common.insight_tables import show_insight_table + + show_insight_table(df, name=key or download_name.replace(" ", "-") or "table") # Show download button with the given download name for the table if name is given if download_name: st.download_button( diff --git a/src/common/insight_tables.py b/src/common/insight_tables.py index d197d44..14cdcad 100644 --- a/src/common/insight_tables.py +++ b/src/common/insight_tables.py @@ -6,13 +6,42 @@ """ import hashlib import shutil +import tempfile from pathlib import Path import pandas as pd import polars as pl +import streamlit as st from openms_insight import Table ROW_FIELD = "row" +# Rows sampled when fingerprinting a dataframe that has no source files. +_FINGERPRINT_SAMPLE_ROWS = 2000 + + +def _fingerprint_dataframe(df: pd.DataFrame) -> str: + """Cheap content hash of a dataframe, for tables computed in the session. + + Hashing every cell on every Streamlit rerun would defeat the purpose for + large tables, so this combines the shape/schema, the column sums and an + evenly spaced sample of rows (always including the first and last row). + """ + digest = hashlib.sha256() + digest.update(repr((df.shape, [str(c) for c in df.columns], [str(t) for t in df.dtypes])).encode()) + digest.update(df.select_dtypes(include="number").sum().to_numpy(dtype="float64").tobytes()) + if len(df): + step = max(len(df) // _FINGERPRINT_SAMPLE_ROWS, 1) + rows = df.iloc[::step].astype(str) + digest.update(pd.util.hash_pandas_object(rows, index=False).values.tobytes()) + digest.update(df.iloc[[-1]].astype(str).to_csv(index=False).encode()) + return digest.hexdigest() + + +def _default_cache_dir() -> Path: + """Cache location for tables that are not tied to a workflow result directory.""" + workspace = st.session_state.get("workspace") + base = Path(workspace) if workspace else Path(tempfile.gettempdir(), "quantms-web") + return base / "insight_tables" def _cache_key(source_files, extra: str) -> str: @@ -31,7 +60,8 @@ def _to_polars(df: pd.DataFrame) -> tuple[pl.DataFrame, list[dict]]: Tabulator reads a "." in a field name as a nested lookup, so such columns get a safe field name and keep their original name as the column title. """ - df = df.reset_index(drop=True) + # A named index usually holds the IDs, so keep it as a column + df = df.reset_index(drop=all(n is None for n in df.index.names)) renames, column_definitions, used = {}, [], {ROW_FIELD} for column in df.columns: field = str(column).replace(".", "_") @@ -55,7 +85,7 @@ def _to_polars(df: pd.DataFrame) -> tuple[pl.DataFrame, list[dict]]: def show_insight_table( df: pd.DataFrame, name: str, - cache_dir, + cache_dir=None, source_files=(), extra_key: str = "", title: str | None = None, @@ -67,12 +97,19 @@ def show_insight_table( The cache is keyed on ``source_files`` (path and mtime) and ``extra_key``; when either changes (a rerun, new group assignments) a new cache replaces the - old one, so the page never shows a previous run's rows. + old one, so the page never shows a previous run's rows. Without + ``source_files`` (a table computed in the session, e.g. a downstream + analysis step) it is keyed on a fingerprint of the table's content instead. + + Insight components request a rerun the first time they render, so don't + draw one only inside ``if st.button(...):``; keep the result in session + state and render it outside the button block. Args: df: Table to show. name: Stable name for this table; must be unique per page. - cache_dir: Directory holding openms-insight caches. + cache_dir: Directory holding openms-insight caches. Defaults to + ``/insight_tables``. source_files: Files the table is derived from. extra_key: Anything else the table depends on (e.g. parameters). title: Title, also used as the CSV download's file name. @@ -80,7 +117,9 @@ def show_insight_table( page_size: Rows per page. **table_kwargs: Passed on to ``Table`` (e.g. ``initial_sort``). """ - cache_dir = Path(cache_dir) + cache_dir = Path(cache_dir) if cache_dir else _default_cache_dir() + if not source_files: + extra_key = f"{extra_key}|{_fingerprint_dataframe(df)}" cache_id = f"{name}_{_cache_key(source_files, extra_key)}" if not (cache_dir / cache_id).is_dir(): diff --git a/tests/test_insight_table.py b/tests/test_insight_table.py new file mode 100644 index 0000000..6730d01 --- /dev/null +++ b/tests/test_insight_table.py @@ -0,0 +1,101 @@ +"""Tests for the OpenMS-Insight table helpers in src/common/insight_tables.py.""" +import numpy as np +import pandas as pd +import polars as pl + +from src.common.insight_tables import ROW_FIELD, _fingerprint_dataframe, _to_polars + + +def _abundance_df(): + return pd.DataFrame( + { + "ProteinName": ["P1", "P2", "P3"], + "m/z": [1.5, np.nan, 3.5], + "sample1[126.1]": [1, 2, 3], + "PeptideSequence": ["AAK", "BBK", "CCK"], + } + ) + + +def test_to_polars_adds_unique_hidden_row_id(): + pl_df, defs = _to_polars(_abundance_df()) + assert pl_df[ROW_FIELD].n_unique() == 3 + assert ROW_FIELD not in [d["field"] for d in defs] + + +def test_to_polars_keeps_titles_and_sanitizes_dotted_fields(): + pl_df, defs = _to_polars(_abundance_df()) + by_title = {d["title"]: d for d in defs} + # Tabulator reads "." in a field name as a nested path, so it is replaced; + # the visible title keeps the original column name. + assert by_title["sample1[126.1]"]["field"] == "sample1[126_1]" + assert by_title["m/z"]["field"] == "m/z" + assert set(d["field"] for d in defs) <= set(pl_df.columns) + + +def test_to_polars_makes_duplicate_column_names_unique(): + df = pd.DataFrame([[1, 2]], columns=["a.b", "a_b"]) + _, defs = _to_polars(df) + assert len({d["field"] for d in defs}) == 2 + + +def test_to_polars_numeric_columns_sort_as_numbers(): + _, defs = _to_polars(_abundance_df()) + by_title = {d["title"]: d for d in defs} + assert by_title["m/z"]["sorter"] == "number" + assert "sorter" not in by_title["ProteinName"] + + +def test_to_polars_turns_named_index_into_column(): + df = _abundance_df().set_index("ProteinName") + pl_df, defs = _to_polars(df) + assert "ProteinName" in pl_df.columns + assert defs[0]["title"] == "ProteinName" + + +def test_to_polars_drops_unnamed_index_after_filtering(): + pl_df, _ = _to_polars(_abundance_df().iloc[[0, 2]]) + assert "index" not in pl_df.columns + + +def test_to_polars_nan_becomes_null_and_mixed_objects_are_stringified(): + df = pd.DataFrame({"x": [1.0, np.nan], "mixed": ["a", 1]}) + pl_df, _ = _to_polars(df) + assert pl_df["x"].null_count() == 1 + assert pl_df["mixed"].dtype == pl.String + + +def test_fingerprint_is_stable_and_detects_changes(): + df = _abundance_df() + assert _fingerprint_dataframe(df) == _fingerprint_dataframe(df.copy()) + + changed_value = df.copy() + changed_value.loc[0, "sample1[126.1]"] = 99 + assert _fingerprint_dataframe(changed_value) != _fingerprint_dataframe(df) + + assert _fingerprint_dataframe(df.iloc[:2]) != _fingerprint_dataframe(df) + assert _fingerprint_dataframe(df.rename(columns={"m/z": "mz"})) != _fingerprint_dataframe(df) + + +def test_fingerprint_handles_duplicate_columns_and_list_cells(): + _fingerprint_dataframe(pd.DataFrame([[1, 2]], columns=["a", "a"])) + _fingerprint_dataframe(pd.DataFrame({"x": [[1, 2], [3]]})) + _fingerprint_dataframe(pd.DataFrame({"x": []})) + + +def test_table_without_source_files_is_cached_by_content(tmp_path, monkeypatch): + from src.common import insight_tables + + rendered = [] + monkeypatch.setattr(insight_tables.Table, "__call__", lambda self, **kw: rendered.append(self._cache_id)) + df = _abundance_df() + insight_tables.show_insight_table(df, "t", tmp_path) + insight_tables.show_insight_table(df.copy(), "t", tmp_path) + assert rendered[0] == rendered[1] + + changed = df.copy() + changed.loc[0, "sample1[126.1]"] = 99 + insight_tables.show_insight_table(changed, "t", tmp_path) + assert rendered[2] != rendered[0] + # The cache of the previous version of the table is removed + assert [p.name for p in tmp_path.iterdir()] == [rendered[2]]