diff --git a/src/EFCore.PG/Infrastructure/Internal/NpgsqlOptionsExtension.cs b/src/EFCore.PG/Infrastructure/Internal/NpgsqlOptionsExtension.cs
index 7ba3f5fec8..e8fefb0945 100644
--- a/src/EFCore.PG/Infrastructure/Internal/NpgsqlOptionsExtension.cs
+++ b/src/EFCore.PG/Infrastructure/Internal/NpgsqlOptionsExtension.cs
@@ -152,6 +152,17 @@ public override RelationalOptionsExtension WithConnection(DbConnection? connecti
         return clone;
     }
 
+    /// <inheritdoc />
+    public override RelationalOptionsExtension WithConnection(DbConnection? connection, bool owned)
+    {
+        var clone = (NpgsqlOptionsExtension)base.WithConnection(connection, owned);
+
+        // A previously configured data source would otherwise take precedence over the supplied connection.
+        clone.DataSource = null;
+
+        return clone;
+    }
+
     /// <summary>
     ///     Returns a copy of the current instance configured with the specified range mapping.
     /// </summary>
diff --git a/src/EFCore.PG/Storage/Internal/NpgsqlRelationalConnection.cs b/src/EFCore.PG/Storage/Internal/NpgsqlRelationalConnection.cs
index 79e6510967..a984b28f6d 100644
--- a/src/EFCore.PG/Storage/Internal/NpgsqlRelationalConnection.cs
+++ b/src/EFCore.PG/Storage/Internal/NpgsqlRelationalConnection.cs
@@ -185,10 +185,11 @@ public virtual INpgsqlRelationalConnection CreateAdminConnection()
             Multiplexing = false
         }.ToString();
 
+        // Cloned connections must be owned so disposing the wrapper also closes and disposes them.
         var adminNpgsqlOptions = _dataSource is not null
-            ? npgsqlOptions.WithConnection(((NpgsqlConnection)CreateDbConnection()).CloneWith(adminConnectionString))
+            ? npgsqlOptions.WithConnection(((NpgsqlConnection)CreateDbConnection()).CloneWith(adminConnectionString), owned: true)
             : npgsqlOptions.Connection is not null
-                ? npgsqlOptions.WithConnection(DbConnection.CloneWith(adminConnectionString))
+                ? npgsqlOptions.WithConnection(DbConnection.CloneWith(adminConnectionString), owned: true)
                 : npgsqlOptions.WithConnectionString(adminConnectionString);
 
         var optionsBuilder = new DbContextOptionsBuilder();
@@ -221,7 +222,7 @@ public virtual NpgsqlRelationalConnection CloneWith(string connectionString)
 
         var relationalOptions = RelationalOptionsExtension.Extract(Dependencies.ContextOptions)
             .WithConnectionString(null)
-            .WithConnection(clonedDbConnection);
+            .WithConnection(clonedDbConnection, owned: true);
 
         var optionsBuilder = new DbContextOptionsBuilder();
         ((IDbContextOptionsBuilderInfrastructure)optionsBuilder).AddOrUpdateExtension(relationalOptions);
diff --git a/test/EFCore.PG.Tests/NpgsqlRelationalConnectionTest.cs b/test/EFCore.PG.Tests/NpgsqlRelationalConnectionTest.cs
index 7f64213381..e18986e779 100644
--- a/test/EFCore.PG.Tests/NpgsqlRelationalConnectionTest.cs
+++ b/test/EFCore.PG.Tests/NpgsqlRelationalConnectionTest.cs
@@ -51,6 +51,26 @@ public void Uses_DbDataSource_from_DbContextOptions()
         Assert.Equal("Host=FakeHost", connection2.ConnectionString);
     }
 
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public void Supplied_connection_replaces_data_source(bool contextOwnsConnection)
+    {
+        using var dataSource = NpgsqlDataSource.Create("Host=FakeHost1");
+        using var connection = new NpgsqlConnection("Host=FakeHost2");
+        var options = new DbContextOptionsBuilder()
+            .UseNpgsql(dataSource)
+            .UseNpgsql(connection, contextOwnsConnection)
+            .Options;
+
+        using var context = new DbContext(options);
+        var relationalConnection = (NpgsqlRelationalConnection)context.GetService<IRelationalConnection>();
+
+        Assert.Null(relationalConnection.DbDataSource);
+        Assert.Same(connection, relationalConnection.DbConnection);
+        Assert.Equal("Host=FakeHost2", relationalConnection.ConnectionString);
+    }
+
     [Fact(Skip = "Passes in isolation, but fails when the entire test suite is run because of #2891")]
     public void Uses_DbDataSource_from_application_service_provider()
     {
@@ -129,6 +149,36 @@ public void Can_create_master_connection_with_connection_string_and_alternate_ad
             master.ConnectionString);
     }
 
+    [Theory]
+    [InlineData(false, false)]
+    [InlineData(false, true)]
+    [InlineData(true, false)]
+    [InlineData(true, true)]
+    public async Task Admin_connection_is_owned(bool async, bool useDataSource)
+    {
+        using var dataSource = NpgsqlDataSource.Create("Host=FakeHost");
+        using var dbConnection = new NpgsqlConnection("Host=FakeHost");
+        var optionsBuilder = new DbContextOptionsBuilder();
+        var options = (useDataSource ? optionsBuilder.UseNpgsql(dataSource) : optionsBuilder.UseNpgsql(dbConnection)).Options;
+        using var context = new DbContext(options);
+        var connection = context.GetService<INpgsqlRelationalConnection>();
+        using var admin = connection.CreateAdminConnection();
+        using var adminDbConnection = admin.DbConnection;
+
+        if (async)
+        {
+            await admin.DisposeAsync();
+        }
+        else
+        {
+            admin.Dispose();
+        }
+
+        Assert.Throws<ObjectDisposedException>(() => adminDbConnection.ConnectionString = "Host=FakeHost");
+        using var originalClone = ((NpgsqlConnection)connection.DbConnection).CloneWith("Host=FakeHost");
+        using var dataSourceConnection = dataSource.CreateConnection();
+    }
+
     [Theory]
     [InlineData("false")]
     [InlineData("False")]
@@ -167,19 +217,34 @@ public void CurrentAmbientTransaction_returns_transaction_with_enlist_enabled(st
         Transaction.Current = null;
     }
 
-    [ConditionalFact]
-    public void CloneWith_with_connection_and_connection_string()
+    [ConditionalTheory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task CloneWith_with_connection_and_connection_string(bool async)
     {
         var services = NpgsqlTestHelpers.Instance.CreateContextServices(
             new DbContextOptionsBuilder()
                 .UseNpgsql("Host=localhost;Database=DummyDatabase")
                 .Options);
 
-        var relationalConnection = (NpgsqlRelationalConnection)services.GetRequiredService<IRelationalConnection>();
+        using var relationalConnection = (NpgsqlRelationalConnection)services.GetRequiredService<IRelationalConnection>();
 
-        var clone = relationalConnection.CloneWith("Host=localhost;Database=DummyDatabase;Application Name=foo");
+        using var clone = relationalConnection.CloneWith("Host=localhost;Database=DummyDatabase;Application Name=foo");
+        using var clonedDbConnection = clone.DbConnection;
 
         Assert.Equal("Host=localhost;Database=DummyDatabase;Application Name=foo", clone.ConnectionString);
+
+        if (async)
+        {
+            await clone.DisposeAsync();
+        }
+        else
+        {
+            clone.Dispose();
+        }
+
+        Assert.Throws<ObjectDisposedException>(() => clonedDbConnection.ConnectionString = "Host=localhost");
+        using var originalClone = relationalConnection.DbConnection.CloneWith("Host=localhost");
     }
 
     public static NpgsqlRelationalConnection CreateConnection(DbContextOptions options = null, DbDataSource dataSource = null)