From 4f77d87de1e94083ed223718f4ffdde70a1daf6b Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:28:09 +0000 Subject: [PATCH 1/3] ssh: register the Databricks %pip magic on serverless environment 5+ The IPython startup script built PipMagicOverrides with its older constructor. On serverless environment 5 the constructor requires a logger, and on environment 6 there is no Py4J entry point, so the call failed and notebooks fell back to IPython's stock %pip. When the constructor takes a logger, build it from the driver connection the same way dbruntime's own kernel does, passing only the keyword arguments the installed runtime accepts. Older runtimes keep the existing path. Co-authored-by: Isaac --- .../ssh/internal/server/jupyter-init.py | 48 ++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/experimental/ssh/internal/server/jupyter-init.py b/experimental/ssh/internal/server/jupyter-init.py index 055a3966584..ce1ac256cf3 100644 --- a/experimental/ssh/internal/server/jupyter-init.py +++ b/experimental/ssh/internal/server/jupyter-init.py @@ -151,25 +151,51 @@ def _register_common_magics(): @_log_exceptions def _register_pip_magics(): """Register the pip magic command parser with IPython.""" + import inspect + from dbruntime import UserNamespaceInitializer from dbruntime.DatasetInfo import UserNamespaceDict from dbruntime.PipMagicOverrides import PipMagicOverrides user_namespace_initializer = UserNamespaceInitializer.getOrCreate() entry_point = user_namespace_initializer.get_spark_entry_point() - user_ns = UserNamespaceDict( - user_namespace_initializer.get_namespace_globals(), - entry_point.getDriverConf(), - entry_point, - ) ip = get_ipython() - try: - # Older DBRs - pip_magic = PipMagicOverrides(entry_point, ip.user_ns["sc"]._conf, user_ns) - except Exception: - # Newer DBRs - pip_magic = PipMagicOverrides(entry_point, user_ns, ip) + params = inspect.signature(PipMagicOverrides).parameters + if "logger" in params: + # Runtimes that take a logger (serverless environment 5+) are migrating off Py4J, so build the + # magic from the driver connection the way dbruntime's own kernel does: on environment 6 the + # entry point is None. The constructor gained keyword arguments over several runtime releases, + # so pass only the ones this runtime accepts. + driver_connection = user_namespace_initializer.get_driver_connection() + driver_conf = driver_connection.runtime_integration_client.getDriverConf() + user_ns = UserNamespaceDict(user_namespace_initializer.get_namespace_globals(), driver_conf, entry_point) + kwargs = { + "orig_pip_magic": ip.magics_manager.magics["line"]["pip"], + "orig_conda_magic": ip.magics_manager.magics["line"]["conda"], + "driver_conf": driver_conf, + "environment_resource_client": driver_connection.environment_resource_client, + "sc_conf": user_namespace_initializer.db_connection.sc_conf, + } + pip_magic = PipMagicOverrides( + entry_point, + user_ns, + ip, + driver_connection.observability_client.getLogger(), + **{name: value for name, value in kwargs.items() if name in params}, + ) + else: + user_ns = UserNamespaceDict( + user_namespace_initializer.get_namespace_globals(), + entry_point.getDriverConf(), + entry_point, + ) + try: + # Older DBRs + pip_magic = PipMagicOverrides(entry_point, ip.user_ns["sc"]._conf, user_ns) + except Exception: + # Newer DBRs + pip_magic = PipMagicOverrides(entry_point, user_ns, ip) ip.register_magics(pip_magic) From e5833986c16fe2cc5869736241f696a4f63faf01 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:46:53 +0000 Subject: [PATCH 2/3] Add changelog entry Co-authored-by: Isaac --- .nextchanges/cli/ssh-jupyter-pip-magics.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/cli/ssh-jupyter-pip-magics.md diff --git a/.nextchanges/cli/ssh-jupyter-pip-magics.md b/.nextchanges/cli/ssh-jupyter-pip-magics.md new file mode 100644 index 00000000000..2c83cc54bb5 --- /dev/null +++ b/.nextchanges/cli/ssh-jupyter-pip-magics.md @@ -0,0 +1 @@ +* Fix `%pip` in Jupyter notebooks opened over `databricks ssh connect` on serverless environment versions 5 and 6, which fell back to the stock IPython magic instead of the Databricks one. ([#7263](https://github.com/databricks/cli/pull/7263)) From 327738afe3c1c4aac95b7518604367a729b53a38 Mon Sep 17 00:00:00 2001 From: Russell Clarey Date: Tue, 3 Nov 2026 16:36:34 +0100 Subject: [PATCH 3/3] Mention dedicated clusters in changelog entry Co-authored-by: Isaac --- .nextchanges/cli/ssh-jupyter-pip-magics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/cli/ssh-jupyter-pip-magics.md b/.nextchanges/cli/ssh-jupyter-pip-magics.md index 2c83cc54bb5..92fdf57595c 100644 --- a/.nextchanges/cli/ssh-jupyter-pip-magics.md +++ b/.nextchanges/cli/ssh-jupyter-pip-magics.md @@ -1 +1 @@ -* Fix `%pip` in Jupyter notebooks opened over `databricks ssh connect` on serverless environment versions 5 and 6, which fell back to the stock IPython magic instead of the Databricks one. ([#7263](https://github.com/databricks/cli/pull/7263)) +* Fix `%pip` in Jupyter notebooks opened over `databricks ssh connect` on serverless environment versions 5 and 6 and on dedicated clusters running recent Databricks Runtime versions, which fell back to the stock IPython magic instead of the Databricks one. ([#7263](https://github.com/databricks/cli/pull/7263))