diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_catalog.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_catalog.py index 603c2811e..71f6c0a81 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_catalog.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_catalog.py @@ -67,6 +67,12 @@ class CatalogMetricAlert: """Expected group-by attributes; ``None`` means the fixture states no expectation.""" granularity: AnomalyDetectionGranularity | None = None """Detection interval for an ANOMALY alert (DAY/WEEK/MONTH/...). Not a date filter.""" + comparison: dict[str, str | int | float | None] | None = None + """Expected period comparison as ``create_metric_alert`` argument names; ``None`` means unasserted.""" + title: str | None = None + """Alert name the user asked for; ``None`` means unasserted.""" + dashboard_id: str | None = None + """Dashboard the alert must be bound to; ``None`` means unasserted.""" @classmethod def from_dict(cls, d: dict) -> CatalogMetricAlert: @@ -87,4 +93,7 @@ def from_dict(cls, d: dict) -> CatalogMetricAlert: filters=d.get("filters"), attributes=d.get("attributes"), granularity=AnomalyDetectionGranularity.parse(d.get("granularity")), + comparison=d.get("comparison"), + title=d.get("title"), + dashboard_id=d.get("dashboard_id"), ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 191037db8..015524453 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -58,6 +58,37 @@ _ALWAYS_TRIGGER_VALUES = {"Every time", "ALWAYS", "not specified"} _RELATIVE_OPERATORS = ("INCREASES_BY", "DECREASES_BY", "CHANGES_BY") +# Fixture spelling of each period-comparison property -> the create_metric_alert argument. +_COMPARISON_ARGS = { + "kind": "comparison_kind", + "periods_ago": "comparison_periods_ago", + "date_dataset_id": "comparison_date_dataset_id", + "date_attribute_id": "comparison_date_attribute_id", + "arithmetic_operator": "arithmetic_operator", +} +# Compared verbatim: object ids are case-sensitive, unlike the enum-valued properties. +_COMPARISON_ID_ARGS = ("comparison_date_dataset_id", "comparison_date_attribute_id") +# What create_metric_alert applies when the argument is left out, per its own contract. A +# metric-to-metric comparison has no periods, so periods_ago defaults only for a period kind. +_COMPARISON_ARG_DEFAULTS: dict[str, object] = {"comparison_periods_ago": 1, "arithmetic_operator": "DIFFERENCE"} +_PERIOD_COMPARISON_KINDS = ("PREVIOUS_PERIOD", "SAME_PERIOD_PRIOR_YEAR") +_NO_COMPARISON_MARKERS = ("none", "no comparison") + +# AlertEvaluation checks, in report order. Every run is scored on the first group; the second only +# when the item states the matching expectation. +_ALERT_CHECKS = ( + "alert_created", + "operator_correct", + "threshold_correct", + "trigger_correct", + "filters_correct", + "metric_correct", + "recipients_correct", + "attributes_correct", + "granularity_correct", +) +_CONDITIONAL_ALERT_CHECKS = ("comparison_correct", "title_correct", "dashboard_correct") + # Expected-recipient spellings that mean the owner of the API token. _CURRENT_USER_RECIPIENT = "me" _CURRENT_USER_MARKERS = {"me", "myself", "current user", "current_user"} @@ -234,6 +265,66 @@ def _check_granularity(expected: CatalogMetricAlert, actual_args: dict) -> bool: return str(actual).strip().upper() == expected.granularity.value +def _canonical_comparison_value(arg: str, value: object) -> str | int | float | None: + """Fold one comparison value so fixture and tool-argument spellings compare equal.""" + if value is None or str(value).strip() == "": + return None + if arg == "comparison_periods_ago": + return _to_number(value) + if arg in _COMPARISON_ID_ARGS: + return str(value).strip() + return str(value).strip().upper() + + +def _check_comparison(expected: CatalogMetricAlert, actual_args: dict) -> bool: + """Compare only the comparison properties the fixture states. + + ``None`` means unasserted. A stated ``comparison_kind`` of ``None`` asserts the alert has + no comparison at all. ``comparison_kind`` is read as ``create_metric_alert`` resolves it: + left out next to a ``compare_metric_id`` it is METRIC. Any other argument the agent left out + on an alert that does compare counts as the value ``create_metric_alert`` applies for it, so + omitting ``comparison_periods_ago`` matches 1 on a period comparison; a METRIC comparison has + no periods, and with no comparison there is nothing to default. + The comparison period itself (DAY, MONTH, ...) is not checked here: it is the granularity + of the comparison date filter, which ``filters`` asserts. + """ + if expected.comparison is None: + return True + kind = actual_args.get("comparison_kind") or ("METRIC" if actual_args.get("compare_metric_id") else None) + for arg, exp_value in expected.comparison.items(): + act_raw = kind if arg == "comparison_kind" else actual_args.get(arg) + defaults_apply = arg != "comparison_periods_ago" or str(kind).upper() in _PERIOD_COMPARISON_KINDS + if act_raw is None and kind is not None and defaults_apply: + act_raw = _COMPARISON_ARG_DEFAULTS.get(arg) + # Only the fixture side of an object id is trimmed: a padded id is a different id. + actual = act_raw if arg in _COMPARISON_ID_ARGS else _canonical_comparison_value(arg, act_raw) + if actual != exp_value: + return False + return True + + +def _check_title(expected: CatalogMetricAlert, actual_args: dict) -> bool: + """Exact match on the alert name when the fixture states one. + + Only the fixture side is trimmed: a name saved with padding is not the name the user asked for. + + An omitted ``title`` fails: ``create_metric_alert`` then saves an auto-generated name, even + when ``automation_id`` carries a slug of the requested one. + """ + if expected.title is None: + return True + actual = actual_args.get("title") + return isinstance(actual, str) and actual == expected.title + + +def _check_dashboard(expected: CatalogMetricAlert, actual_args: dict) -> bool: + """Compare the bound dashboard id when the fixture states one; the actual id verbatim.""" + if expected.dashboard_id is None: + return True + actual = actual_args.get("dashboard_id") + return isinstance(actual, str) and actual == expected.dashboard_id + + def _check_metric(expected: CatalogMetricAlert, actual_args: dict) -> bool: if not expected.metric_id: return True @@ -489,6 +580,15 @@ def generate_simulated_alert_response( original_request = f'Your original request to the agent was: "{question}"\n' if question else "" + # Only a stated title is enforced: an unstated one is the agent's to choose, and telling the + # sim-user nothing keeps it from objecting to a generated name. + title_rule = ( + f'8. The alert must be named exactly "{expected.title}". If a summary or proposal shows a ' + "different name, do NOT confirm: state that exact name and ask the agent to use it.\n" + if expected.title + else "" + ) + system_prompt = ( "You are a user requesting creation of an alert for a metric from an AI agent. " "Respond naturally but always steer toward the exact values you were given.\n" @@ -509,7 +609,10 @@ def generate_simulated_alert_response( + recipients_rule + filters_rule + f"6. Proactively state how often you want to be alerted in your first reply: {trigger_request}. " - " Repeat it if the agent proposes a different cadence.\n" + period_rule + "Reply concisely and directly." + " Repeat it if the agent proposes a different cadence.\n" + + period_rule + + title_rule + + "Reply concisely and directly." ) messages: list = [{"role": "system", "content": system_prompt}] @@ -550,22 +653,28 @@ class AlertEvaluation: recipients_correct: bool attributes_correct: bool = True granularity_correct: bool = True + # ``None``: the item states no such expectation, so the check is neither scored nor gated. + comparison_correct: bool | None = None + title_correct: bool | None = None + dashboard_correct: bool | None = None + + def checks(self) -> dict[str, bool]: + """Every check that ran, by name; strict_pass requires all of them. + + The conditional checks are left out when unasserted, so an item that names no title does + not report a ``title_correct`` it was never scored on, and a refusal on it is not counted + as a title failure. + """ + ran: dict[str, bool] = {name: getattr(self, name) for name in _ALERT_CHECKS} + for name in _CONDITIONAL_ALERT_CHECKS: + value = getattr(self, name) + if value is not None: + ran[name] = value + return ran @property def strict_pass(self) -> bool: - return all( - [ - self.alert_created, - self.operator_correct, - self.threshold_correct, - self.trigger_correct, - self.filters_correct, - self.metric_correct, - self.recipients_correct, - self.attributes_correct, - self.granularity_correct, - ] - ) + return all(self.checks().values()) @dataclass @@ -666,6 +775,68 @@ def _normalize_expected_attributes(expected: dict) -> list | None: raise ValueError(f"Attributes expectation must be a list or a display string, got {type(attributes).__name__}") +def _normalize_expected_comparison(expected: dict) -> dict[str, str | int | float | None] | None: + """ + * ``Comparison`` object -> its stated properties, keyed by tool-argument name + * "None" / "no comparison" -> ``{"comparison_kind": None}`` (stated: no comparison) + * absent, empty, or prose -> ``None`` (unstated; comparison not asserted) + + Each property of the object is asserted only when it carries a value, so an item can pin + the comparison kind without also pinning the date dataset the agent may pick. + """ + comparison = _case_insensitive_get(expected, "comparison") + if comparison is None: + return None + if isinstance(comparison, str): + return {"comparison_kind": None} if any(kw in comparison.lower() for kw in _NO_COMPARISON_MARKERS) else None + if not isinstance(comparison, dict): + raise ValueError(f"Comparison expectation must be an object or a display string, got {comparison!r}") + stated: dict[str, str | int | float | None] = {} + for key, value in comparison.items(): + name = str(key).strip().lower() + arg = _COMPARISON_ARGS.get(name) or (name if name in _COMPARISON_ARGS.values() else None) + if arg is None: + # Validated here so a misspelled property fails before the run spends an API call. + raise ValueError(f"Unrecognised comparison property {key!r}; expected one of {sorted(_COMPARISON_ARGS)}") + canonical = _canonical_comparison_value(arg, value) + if canonical is None and arg == "comparison_periods_ago" and str(value or "").strip(): + raise ValueError(f"Comparison periods_ago must be a number, got {value!r}") + if canonical is not None: + stated[arg] = canonical + return stated or None + + +def _normalize_expected_text(expected: dict, *keys: str) -> str | None: + """A stated string expectation, or ``None`` when absent or blank (unasserted).""" + value = _case_insensitive_get(expected, *keys) + if value is None: + return None + if not isinstance(value, str): + raise ValueError(f"{keys[0]} expectation must be a string, got {value!r}") + return value.strip() or None + + +def _normalize_expected_dashboard(expected: dict) -> str | None: + """ + * ``dashboard_id`` -> that id + * ``Dashboard`` "Title (id)" -> the id in the trailing parentheses, as ``Metric`` is read + * absent / empty -> ``None`` (unstated; binding not asserted) + + A ``Dashboard`` without a trailing ``(id)`` raises: a title alone can never equal the + ``dashboard_id`` the tool receives, so the item would fail on every run. + """ + dashboard_id = _normalize_expected_text(expected, "dashboard_id") + if dashboard_id is not None: + return dashboard_id + display = _normalize_expected_text(expected, "dashboard") + if display is None: + return None + parsed = _parse_metric_id(display) + if parsed is None: + raise ValueError(f'Dashboard expectation must be "Title (id)", got {display!r}; use dashboard_id for a bare id') + return parsed + + def _normalize_expected_output(expected: dict) -> CatalogMetricAlert: """Parse expected_output dict into CatalogMetricAlert, accepting display-format or internal-format keys.""" operator = _case_insensitive_get(expected, "operator") or "GREATER_THAN" @@ -696,6 +867,9 @@ def _normalize_expected_output(expected: dict) -> CatalogMetricAlert: granularity = AnomalyDetectionGranularity.parse( _case_insensitive_get(expected, "granularity", "detection granularity") ) + comparison = _normalize_expected_comparison(expected) + title = _normalize_expected_text(expected, "title") + dashboard_id = _normalize_expected_dashboard(expected) return CatalogMetricAlert( operator=operator, @@ -708,6 +882,9 @@ def _normalize_expected_output(expected: dict) -> CatalogMetricAlert: filters=filters, attributes=attributes, granularity=granularity, + comparison=comparison, + title=title, + dashboard_id=dashboard_id, ) @@ -893,6 +1070,13 @@ def _run_once(conv_id: str) -> AlertRunResult: and _check_recipients(expected, actual_args, sdk=sdk, current_user_id=current_user_id), attributes_correct=tool_called and _check_attributes(expected, actual_args), granularity_correct=tool_called and _check_granularity(expected, actual_args), + comparison_correct=None + if expected.comparison is None + else tool_called and _check_comparison(expected, actual_args), + title_correct=None if expected.title is None else tool_called and _check_title(expected, actual_args), + dashboard_correct=None + if expected.dashboard_id is None + else tool_called and _check_dashboard(expected, actual_args), ) return AlertRunResult( conversation_id=conv_id, @@ -934,19 +1118,7 @@ def _run_once(conv_id: str) -> AlertRunResult: pass_power_k = all(r.eval.strict_pass for r in run_results) best = max( run_results, - key=lambda r: sum( - [ - r.eval.alert_created, - r.eval.operator_correct, - r.eval.threshold_correct, - r.eval.trigger_correct, - r.eval.filters_correct, - r.eval.metric_correct, - r.eval.recipients_correct, - r.eval.attributes_correct, - r.eval.granularity_correct, - ] - ), + key=lambda r: sum(r.eval.checks().values()), ) return AgenticAlertSummary( run_results=run_results, @@ -966,17 +1138,8 @@ def _run_detail(run: AlertRunResult) -> dict: Extracted so a failing run is described by exactly the same keys as the winning run -- for this kind that means the full per-field check breakdown and the arguments the run sent. """ - ev = run.eval return { - "alert_created": ev.alert_created, - "operator_correct": ev.operator_correct, - "threshold_correct": ev.threshold_correct, - "trigger_correct": ev.trigger_correct, - "filters_correct": ev.filters_correct, - "metric_correct": ev.metric_correct, - "recipients_correct": ev.recipients_correct, - "attributes_correct": ev.attributes_correct, - "granularity_correct": ev.granularity_correct, + **run.eval.checks(), "actual_alert_arguments": run.actual_alert_arguments, # Why the loop stopped. alert_created=False alone cannot tell a refusal from a run # that hit max_iterations while still on track -- see LoopExit. Per-run rather than @@ -1052,18 +1215,7 @@ def _write_scores(ctx: RunTraceContext) -> None: for run_idx, run in enumerate(summary.run_results): pt = ctx.trace(run.conversation_id) - ev = run.eval - strict_checks = { - "alert_created": ev.alert_created, - "operator_correct": ev.operator_correct, - "threshold_correct": ev.threshold_correct, - "trigger_correct": ev.trigger_correct, - "filters_correct": ev.filters_correct, - "metric_correct": ev.metric_correct, - "recipients_correct": ev.recipients_correct, - "attributes_correct": ev.attributes_correct, - "granularity_correct": ev.granularity_correct, - } + strict_checks = run.eval.checks() with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") @@ -1112,13 +1264,8 @@ def _write_scores(ctx: RunTraceContext) -> None: gate_note = gate_failure_note(gate, runs_passed, runs_effective) exc = AlertSkillAssertionError( f"Alert skill assertion failed. {gate_note} strict_pass={ev.strict_pass}. " - f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, " - f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, " - f"filters_correct={ev.filters_correct}, metric_correct={ev.metric_correct}, " - f"recipients_correct={ev.recipients_correct}, " - f"attributes_correct={ev.attributes_correct}, " - f"granularity_correct={ev.granularity_correct}. " - f"Actual args: {best.actual_alert_arguments}" + + ", ".join(f"{name}={value}" for name, value in ev.checks().items()) + + f". Actual args: {best.actual_alert_arguments}" ) exc.reasoning_steps = best.reasoning_steps exc.conversation_id = best.conversation_id diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index ea3aa59a1..3cf4dd79d 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -9,9 +9,12 @@ AlertEvaluation, AlertSkillAssertionError, _check_attributes, + _check_comparison, + _check_dashboard, _check_filters, _check_granularity, _check_recipients, + _check_title, _check_trigger, _deep_subset, _normalize_expected_output, @@ -1220,6 +1223,292 @@ def test_every_gen_ai_interval_is_accepted(): assert AnomalyDetectionGranularity.parse(" ") is None +# --- Period comparison ------------------------------------------------------------------ + +# What create_metric_alert receives for "decreases by more than 15% compared with the previous +# period": the comparison properties plus the date filter that sets the compared period. +_PREVIOUS_PERIOD_ARGS = { + "metric_id": "net_sales", + "operator": "DECREASES_BY", + "threshold": 0.15, + "arithmetic_operator": "CHANGE", + "comparison_kind": "PREVIOUS_PERIOD", + "comparison_date_dataset_id": "date", + "comparison_periods_ago": 1, + "filters": [_DATE_FILTER], +} +_PREVIOUS_PERIOD_EXPECTATION = { + "Operator": "DECREASES_BY", + "Threshold": "0.15", + "Comparison": {"kind": "PREVIOUS_PERIOD", "periods_ago": 1, "arithmetic_operator": "CHANGE"}, +} + + +def test_comparison_matching_the_request_passes() -> None: + expected = _normalize_expected_output(_PREVIOUS_PERIOD_EXPECTATION) + assert _check_comparison(expected, _PREVIOUS_PERIOD_ARGS) is True + + +@pytest.mark.parametrize( + "wrong_args", + [ + {"comparison_kind": "SAME_PERIOD_PRIOR_YEAR"}, + {"comparison_periods_ago": 2}, + {"arithmetic_operator": "DIFFERENCE"}, + # A hand-made "previous period" metric instead of a comparison. + {"comparison_kind": None, "compare_metric_id": "net_sales_previous_period"}, + ], +) +def test_comparison_differing_from_the_request_fails(wrong_args: dict) -> None: + expected = _normalize_expected_output(_PREVIOUS_PERIOD_EXPECTATION) + assert _check_comparison(expected, {**_PREVIOUS_PERIOD_ARGS, **wrong_args}) is False + + +def test_omitted_comparison_arguments_count_as_the_tool_defaults() -> None: + expected = _normalize_expected_output( + {"Comparison": {"kind": "PREVIOUS_PERIOD", "periods_ago": 1, "arithmetic_operator": "DIFFERENCE"}} + ) + assert _check_comparison(expected, {"comparison_kind": "PREVIOUS_PERIOD"}) is True + assert _check_comparison(expected, {"comparison_kind": "PREVIOUS_PERIOD", "arithmetic_operator": None}) is True + + +def test_only_the_stated_comparison_properties_are_asserted() -> None: + expected = _normalize_expected_output({"Comparison": {"kind": "previous_period", "date_dataset_id": ""}}) + assert expected.comparison == {"comparison_kind": "PREVIOUS_PERIOD"} + assert _check_comparison(expected, {**_PREVIOUS_PERIOD_ARGS, "comparison_periods_ago": 3}) is True + + +def test_comparison_accepts_tool_argument_spellings() -> None: + expected = _normalize_expected_output( + {"comparison": {"comparison_kind": "PREVIOUS_PERIOD", "date_dataset_id": "date"}} + ) + assert expected.comparison == {"comparison_kind": "PREVIOUS_PERIOD", "comparison_date_dataset_id": "date"} + assert _check_comparison(expected, _PREVIOUS_PERIOD_ARGS) is True + assert _check_comparison(expected, {**_PREVIOUS_PERIOD_ARGS, "comparison_date_dataset_id": "order_date"}) is False + assert _check_comparison(expected, {**_PREVIOUS_PERIOD_ARGS, "comparison_date_dataset_id": " date "}) is False + + +@pytest.mark.parametrize("unstated", [{}, {"Comparison": ""}, {"Comparison": {}}, {"Comparison": "vs last month"}]) +def test_comparison_is_unasserted_when_the_fixture_states_none(unstated: dict) -> None: + expected = _normalize_expected_output({"Operator": "DECREASES_BY", **unstated}) + assert expected.comparison is None + assert _check_comparison(expected, {}) is True + assert _check_comparison(expected, _PREVIOUS_PERIOD_ARGS) is True + + +@pytest.mark.parametrize("marker", ["None", "no comparison"]) +def test_a_none_comparison_rejects_any_period_comparison(marker: str) -> None: + expected = _normalize_expected_output({"Operator": "GREATER_THAN", "Comparison": marker}) + assert _check_comparison(expected, {"operator": "GREATER_THAN"}) is True + assert _check_comparison(expected, _PREVIOUS_PERIOD_ARGS) is False + + +@pytest.mark.parametrize( + "bad", + [ + {"Comparison": {"period": "MONTH"}}, + {"Comparison": ["PREVIOUS_PERIOD"]}, + {"Comparison": {"kind": "PREVIOUS_PERIOD", "periods_ago": "one"}}, + ], +) +def test_a_malformed_comparison_expectation_fails_before_the_run(bad: dict) -> None: + with pytest.raises(ValueError, match="omparison"): + _normalize_expected_output(bad) + + +def test_a_none_comparison_display_string_with_a_note_is_still_stated() -> None: + expected = _normalize_expected_output({"Comparison": "None (no period comparison)"}) + assert expected.comparison == {"comparison_kind": None} + + +def test_a_metric_comparison_is_read_the_same_whether_its_kind_is_spelled_or_defaulted() -> None: + implicit = {"operator": "INCREASES_BY", "compare_metric_id": "budget"} + explicit = {**implicit, "comparison_kind": "METRIC"} + as_metric = _normalize_expected_output({"Comparison": {"kind": "METRIC"}}) + no_comparison = _normalize_expected_output({"Comparison": "None"}) + assert _check_comparison(as_metric, implicit) is True + assert _check_comparison(as_metric, explicit) is True + assert _check_comparison(no_comparison, implicit) is False + assert _check_comparison(no_comparison, explicit) is False + + +def test_tool_defaults_do_not_stand_in_for_a_missing_comparison() -> None: + expected = _normalize_expected_output({"Comparison": {"periods_ago": 1}}) + assert _check_comparison(expected, {"operator": "GREATER_THAN"}) is False + assert _check_comparison(expected, {"comparison_kind": "PREVIOUS_PERIOD"}) is True + + +def test_a_metric_comparison_gets_no_periods_ago_default() -> None: + """A hand-made "previous period" metric must not satisfy a stated periods_ago.""" + expected = _normalize_expected_output({"Comparison": {"periods_ago": 1}}) + assert _check_comparison(expected, {"compare_metric_id": "net_sales_previous_period"}) is False + assert _check_comparison(expected, {"comparison_kind": "SAME_PERIOD_PRIOR_YEAR"}) is True + # arithmetic_operator does default on a METRIC comparison: the tool applies it to every relative alert. + arithmetic = _normalize_expected_output({"Comparison": {"arithmetic_operator": "DIFFERENCE"}}) + assert _check_comparison(arithmetic, {"compare_metric_id": "budget"}) is True + + +def test_same_period_prior_year_date_attribute_can_be_asserted() -> None: + expected = _normalize_expected_output( + {"Comparison": {"kind": "SAME_PERIOD_PRIOR_YEAR", "date_attribute_id": "date.year"}} + ) + args = {"comparison_kind": "SAME_PERIOD_PRIOR_YEAR", "comparison_date_attribute_id": "date.year"} + assert _check_comparison(expected, args) is True + assert _check_comparison(expected, {**args, "comparison_date_attribute_id": "date.month"}) is False + + +def test_a_mismatched_comparison_fails_strict_pass() -> None: + ev = AlertEvaluation( + alert_created=True, + operator_correct=True, + threshold_correct=True, + trigger_correct=True, + filters_correct=True, + metric_correct=True, + recipients_correct=True, + comparison_correct=False, + ) + assert ev.strict_pass is False + + +# --- Title and dashboard binding -------------------------------------------------------- + +_SALES_DASHBOARD_ID = "bf439696-d6c6-4d41-a102-dd98e2f3da35" + + +def test_requested_title_is_asserted_exactly() -> None: + expected = _normalize_expected_output({"Title": " QA margin watch qz7 "}) + assert expected.title == "QA margin watch qz7" + assert _check_title(expected, {"title": "QA margin watch qz7"}) is True + assert _check_title(expected, {"title": "Gross Profit Margin Alert"}) is False + assert _check_title(expected, {"title": "qa margin watch qz7"}) is False + assert _check_title(expected, {"title": " QA margin watch qz7 "}) is False + + +def test_a_title_surviving_only_in_the_automation_id_fails() -> None: + expected = _normalize_expected_output({"Title": "QA margin watch qz7"}) + assert _check_title(expected, {"automation_id": "qa-margin-watch-qz7"}) is False + assert _check_title(expected, {"automation_id": "qa-margin-watch-qz7", "title": None}) is False + + +@pytest.mark.parametrize("unstated", [{}, {"Title": ""}, {"Title": " "}]) +def test_title_is_unasserted_when_the_fixture_states_none(unstated: dict) -> None: + expected = _normalize_expected_output(unstated) + assert expected.title is None + assert _check_title(expected, {"title": "Anything"}) is True + assert _check_title(expected, {}) is True + + +@pytest.mark.parametrize( + "stated", + [ + {"Dashboard": f"2. Sales ({_SALES_DASHBOARD_ID})"}, + {"dashboard_id": _SALES_DASHBOARD_ID}, + ], +) +def test_dashboard_binding_is_asserted_by_id(stated: dict) -> None: + expected = _normalize_expected_output(stated) + assert expected.dashboard_id == _SALES_DASHBOARD_ID + assert _check_dashboard(expected, {"dashboard_id": _SALES_DASHBOARD_ID}) is True + assert _check_dashboard(expected, {"dashboard_id": "092929af-375a-4e9c-964f-2add8cdbd259"}) is False + assert _check_dashboard(expected, {"dashboard_id": f" {_SALES_DASHBOARD_ID} "}) is False + assert _check_dashboard(expected, {}) is False + + +@pytest.mark.parametrize("unstated", [{}, {"Dashboard": ""}, {"dashboard_id": None}]) +def test_dashboard_is_unasserted_when_the_fixture_states_none(unstated: dict) -> None: + expected = _normalize_expected_output(unstated) + assert expected.dashboard_id is None + assert _check_dashboard(expected, {"dashboard_id": _SALES_DASHBOARD_ID}) is True + assert _check_dashboard(expected, {}) is True + + +@pytest.mark.parametrize("bad", [{"Title": 7}, {"Dashboard": ["2. Sales"]}]) +def test_a_non_string_title_or_dashboard_expectation_fails_before_the_run(bad: dict) -> None: + with pytest.raises(ValueError, match="must be a string"): + _normalize_expected_output(bad) + + +@pytest.mark.parametrize("display", ["2. Sales", _SALES_DASHBOARD_ID]) +def test_a_dashboard_without_an_id_fails_before_the_run(display: str) -> None: + with pytest.raises(ValueError, match="Title \\(id\\)"): + _normalize_expected_output({"Dashboard": display}) + + +def test_unasserted_conditional_checks_are_neither_reported_nor_gated() -> None: + refused = AlertEvaluation( + alert_created=False, + operator_correct=False, + threshold_correct=False, + trigger_correct=False, + filters_correct=False, + metric_correct=False, + recipients_correct=False, + ) + assert not {"comparison_correct", "title_correct", "dashboard_correct"} & refused.checks().keys() + passed = AlertEvaluation( + alert_created=True, + operator_correct=True, + threshold_correct=True, + trigger_correct=True, + filters_correct=True, + metric_correct=True, + recipients_correct=True, + title_correct=True, + ) + assert passed.checks()["title_correct"] is True + assert "dashboard_correct" not in passed.checks() + assert passed.strict_pass is True + + +def test_a_refusal_fails_a_stated_title_and_reports_no_unstated_check() -> None: + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _no_alert_chat_result() + + with _patched(mock_client): + summary = run_agentic_alert_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create an alert titled 'QA margin watch qz7'", + expected_output={"operator": "LESS_THAN", "threshold": 0.2, "Title": "QA margin watch qz7"}, + k=1, + max_iterations=1, + ) + + checks = summary.best.eval.checks() + assert checks["title_correct"] is False + assert "comparison_correct" not in checks + assert "dashboard_correct" not in checks + + +@pytest.mark.parametrize("field", ["title_correct", "dashboard_correct"]) +def test_a_wrong_title_or_dashboard_fails_strict_pass(field: str) -> None: + ev = AlertEvaluation( + alert_created=True, + operator_correct=True, + threshold_correct=True, + trigger_correct=True, + filters_correct=True, + metric_correct=True, + recipients_correct=True, + **{field: False}, + ) + assert ev.strict_pass is False + + +def test_sim_user_insists_on_a_stated_title() -> None: + prompt = _sim_user_prompt({"Operator": "LESS_THAN", "Title": "QA margin watch qz7"}) + assert 'named exactly "QA margin watch qz7"' in prompt + + +def test_sim_user_is_not_told_a_title_or_dashboard_the_fixture_leaves_open() -> None: + prompt = _sim_user_prompt({"Operator": "LESS_THAN", "Dashboard": f"2. Sales ({_SALES_DASHBOARD_ID})"}) + assert "named exactly" not in prompt + assert _SALES_DASHBOARD_ID not in prompt + + # --- LoopExit: why the simulated-user loop stopped ------------------------------------------ # alert_created=False is reached three different ways, and before exit_reason existed they # were indistinguishable in the result. Every downstream check is `alert_created and ...`, so