Make an explicitly selected profile take precedence over auth environment variables

NEXT_CHANGELOG.md

@@ -6,6 +6,8 @@
 
 ### CLI
 
+* An explicitly selected profile (`--profile` or a bundle's `workspace.profile`) now takes precedence over auth environment variables (`DATABRICKS_HOST`, `DATABRICKS_TOKEN`, etc.) instead of being silently shadowed by them; env vars still fill auth fields the profile leaves empty ([#5096](https://github.com/databricks/cli/issues/5096)).
+
 ### Bundles
 
 ### Dependency updates

acceptance/cmd/api/default-profile-vs-env/output.txt

@@ -0,0 +1,22 @@
+
+=== api without --profile uses env auth, ignoring default_profile (#5616)
+
+>>> [CLI] api get /api/2.0/clusters/list
+{}
+
+>>> print_requests.py --get //api/2.0/clusters/list
+{
+  "headers": {
+    "Authorization": [
+      "Bearer [DATABRICKS_TOKEN]"
+    ],
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/api_get cmd-exec-id/[UUID] interactive/none auth/pat"
+    ],
+    "X-Databricks-Workspace-Id": [
+      "[NUMID]"
+    ]
+  },
+  "method": "GET",
+  "path": "/api/2.0/clusters/list"
+}

acceptance/cmd/api/default-profile-vs-env/script

@@ -0,0 +1,19 @@
+sethome "./home"
+
+# A default profile with conflicting (basic) auth on a different host. The env
+# below still points a PAT at the test server. Without the #5616 guard, the
+# default profile would be pinned and merged with the env PAT, failing with
+# "more than one authorization method configured".
+cat > "./home/.databrickscfg" <<EOF
+[__settings__]
+default_profile = other
+
+[other]
+host     = https://other.cloud.databricks.test
+username = user
+password = pass
+EOF
+
+title "api without --profile uses env auth, ignoring default_profile (#5616)\n"
+MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list
+trace print_requests.py --get //api/2.0/clusters/list

acceptance/cmd/api/default-profile-vs-env/test.toml

@@ -0,0 +1,3 @@
+Ignore = [
+    "home",
+]

acceptance/cmd/api/profile-overrides-env/output.txt

@@ -0,0 +1,44 @@
+
+=== api --profile overrides auth env vars (#5096)
+
+>>> [CLI] api get /api/2.0/clusters/list --profile my-workspace
+{}
+
+>>> print_requests.py --get //api/2.0/clusters/list
+{
+  "headers": {
+    "Authorization": [
+      "Bearer [DATABRICKS_TOKEN]"
+    ],
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/api_get cmd-exec-id/[UUID] interactive/none auth/pat"
+    ],
+    "X-Databricks-Workspace-Id": [
+      "[NUMID]"
+    ]
+  },
+  "method": "GET",
+  "path": "/api/2.0/clusters/list"
+}
+
+=== api host-only --profile fills the token from the environment (#5096)
+
+>>> [CLI] api get /api/2.0/clusters/list --profile host-only
+{}
+
+>>> print_requests.py --get //api/2.0/clusters/list
+{
+  "headers": {
+    "Authorization": [
+      "Bearer [DATABRICKS_TOKEN]"
+    ],
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/api_get cmd-exec-id/[UUID] interactive/none auth/pat"
+    ],
+    "X-Databricks-Workspace-Id": [
+      "[NUMID]"
+    ]
+  },
+  "method": "GET",
+  "path": "/api/2.0/clusters/list"
+}

acceptance/cmd/api/profile-overrides-env/script

@@ -0,0 +1,29 @@
+sethome "./home"
+
+# One profile with full credentials, one host-only; both point at the test
+# server while the auth env vars below point elsewhere.
+cat > "./home/.databrickscfg" <<EOF
+[my-workspace]
+host  = $DATABRICKS_HOST
+token = $DATABRICKS_TOKEN
+
+[host-only]
+host  = $DATABRICKS_HOST
+EOF
+
+# direnv-style auth env vars for a different workspace; before #5096 these
+# shadowed the profile selected with --profile.
+real_token=$DATABRICKS_TOKEN
+export DATABRICKS_HOST=https://dev.cloud.databricks.test
+export DATABRICKS_TOKEN=dev-token
+
+title "api --profile overrides auth env vars (#5096)\n"
+MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list --profile my-workspace
+trace print_requests.py --get //api/2.0/clusters/list
+
+# Host-only profile: the profile wins for the host, but env fills the token it
+# omits (#5096).
+export DATABRICKS_TOKEN=$real_token
+title "api host-only --profile fills the token from the environment (#5096)\n"
+MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list --profile host-only
+trace print_requests.py --get //api/2.0/clusters/list

acceptance/cmd/api/profile-overrides-env/test.toml

@@ -0,0 +1,3 @@
+Ignore = [
+    "home",
+]

acceptance/cmd/auth/describe/account-host-with-workspace-id/output.txt

@@ -7,10 +7,10 @@ Account ID: acct-123
 Authenticated with: pat
 -----
 Current configuration:
-  ✓ host: [DATABRICKS_URL] (from DATABRICKS_HOST environment variable)
+  ✓ host: [DATABRICKS_URL] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
   ✓ account_id: acct-123 (from [TEST_TMP_DIR]/home/.databrickscfg config file)
   ✓ workspace_id: [NUMID] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
-  ✓ token: ******** (from DATABRICKS_TOKEN environment variable)
+  ✓ token: ******** (from [TEST_TMP_DIR]/home/.databrickscfg config file)
   ✓ profile: acct-with-ws (from --profile flag)
   ✓ databricks_cli_path: [CLI]
   ✓ auth_type: pat

acceptance/cmd/auth/describe/profile-overrides-env/output.txt

@@ -0,0 +1,36 @@
+
+=== Describe with --profile overrides auth env vars (#5096)
+
+>>> [CLI] auth describe --profile my-workspace
+Host: [DATABRICKS_URL]
+User: [USERNAME]
+Authenticated with: pat
+-----
+Current configuration:
+  ✓ host: [DATABRICKS_URL] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+  ✓ workspace_id: [NUMID]
+  ✓ token: ******** (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+  ✓ profile: my-workspace (from --profile flag)
+  ✓ databricks_cli_path: [CLI]
+  ✓ auth_type: pat
+  ✓ rate_limit: [NUMID] (from DATABRICKS_RATE_LIMIT environment variable)
+  ✓ cloud: AWS
+  ✓ discovery_url: [DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server
+
+=== Describe with a host-only --profile fills the token from the environment (#5096)
+
+>>> [CLI] auth describe --profile host-only
+Host: [DATABRICKS_URL]
+User: [USERNAME]
+Authenticated with: pat
+-----
+Current configuration:
+  ✓ host: [DATABRICKS_URL] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+  ✓ workspace_id: [NUMID]
+  ✓ token: ******** (from DATABRICKS_TOKEN environment variable)
+  ✓ profile: host-only (from --profile flag)
+  ✓ databricks_cli_path: [CLI]
+  ✓ auth_type: pat
+  ✓ rate_limit: [NUMID] (from DATABRICKS_RATE_LIMIT environment variable)
+  ✓ cloud: AWS
+  ✓ discovery_url: [DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server

acceptance/cmd/auth/describe/profile-overrides-env/script

@@ -0,0 +1,26 @@
+sethome "./home"
+
+# A profile carries full credentials; a second profile carries only a host.
+cat > "./home/.databrickscfg" <<EOF
+[my-workspace]
+host  = $DATABRICKS_HOST
+token = $DATABRICKS_TOKEN
+
+[host-only]
+host  = $DATABRICKS_HOST
+EOF
+
+# direnv-style auth env vars for a different workspace; before #5096 these
+# shadowed the profile selected with --profile.
+real_token=$DATABRICKS_TOKEN
+export DATABRICKS_HOST=https://dev.cloud.databricks.test
+export DATABRICKS_TOKEN=dev-token
+
+title "Describe with --profile overrides auth env vars (#5096)\n"
+trace $CLI auth describe --profile my-workspace
+
+# Host-only profile: the profile wins for the host, but env fills the token it
+# omits (#5096).
+export DATABRICKS_TOKEN=$real_token
+title "Describe with a host-only --profile fills the token from the environment (#5096)\n"
+trace $CLI auth describe --profile host-only

acceptance/cmd/auth/describe/profile-overrides-env/test.toml

@@ -0,0 +1,3 @@
+Ignore = [
+    "home"
+]

bundle/config/workspace.go

@@ -182,9 +182,14 @@ func (w *Workspace) Client(ctx context.Context) (*databricks.WorkspaceClient, er
 
 	cfg := w.Config(ctx)
 
-	// If only the host is configured, we try and unambiguously match it to
-	// a profile in the user's databrickscfg file. Override the default loaders.
-	if w.Host != "" && w.Profile == "" {
+	switch {
+	case w.Profile != "":
+		// An explicit profile wins over auth env vars (#5096).
+		// ValidateConfigAndProfileHost below still checks host agreement.
+		cfg.Loaders = databrickscfg.ProfileAuthLoaders
+	case w.Host != "":
+		// If only the host is configured, we try and unambiguously match it to
+		// a profile in the user's databrickscfg file. Override the default loaders.
 		cfg.Loaders = []config.Loader{
 			// Load auth creds from env vars
 			config.ConfigAttributes,

bundle/config/workspace_test.go

@@ -155,6 +155,78 @@ func TestWorkspaceClientNormalizesHostBeforeProfileResolution(t *testing.T) {
 	assert.Equal(t, "ws2", client.Config.Profile)
 }
 
+func TestWorkspaceClientProfileOverridesAuthEnv(t *testing.T) {
+	// An explicit profile must win over auth env vars (#5096).
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+		Profile: "tst",
+		Host:    "https://tst.cloud.databricks.test",
+		Token:   "tst-token",
+	})
+	require.NoError(t, err)
+
+	// direnv-style auth env vars pointing at a different (dev) workspace.
+	t.Setenv("DATABRICKS_HOST", "https://dev.cloud.databricks.test")
+	t.Setenv("DATABRICKS_TOKEN", "dev-token")
+
+	w := Workspace{Profile: "tst"}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "tst", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	assert.Equal(t, "tst-token", client.Config.Token)
+}
+
+func TestWorkspaceClientProfileFillsAuthFromEnv(t *testing.T) {
+	// Host-only profile: the profile wins for the host, but env fills the token
+	// it omits (#5096).
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+		Profile: "host-only",
+		Host:    "https://tst.cloud.databricks.test",
+	})
+	require.NoError(t, err)
+
+	t.Setenv("DATABRICKS_TOKEN", "env-token")
+
+	w := Workspace{Profile: "host-only"}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "host-only", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	// The token is not in the profile, so it is filled from the environment.
+	assert.Equal(t, "env-token", client.Config.Token)
+}
+
+func TestWorkspaceClientHostAndProfileOverridesAuthEnv(t *testing.T) {
+	// Bundle pins both workspace.host and workspace.profile: the profile wins
+	// for auth and the host check passes because they agree (#5096).
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+		Profile: "tst",
+		Host:    "https://tst.cloud.databricks.test",
+		Token:   "tst-token",
+	})
+	require.NoError(t, err)
+
+	// direnv-style auth env vars pointing at a different (dev) workspace.
+	t.Setenv("DATABRICKS_HOST", "https://dev.cloud.databricks.test")
+	t.Setenv("DATABRICKS_TOKEN", "dev-token")
+
+	w := Workspace{
+		Host:    "https://tst.cloud.databricks.test",
+		Profile: "tst",
+	}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "tst", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	assert.Equal(t, "tst-token", client.Config.Token)
+}
+
 func TestWorkspaceConfigHTTPTimeout(t *testing.T) {
 	for _, tc := range []struct {
 		envVal string

cmd/api/api.go

@@ -80,22 +80,27 @@ func makeCommand(method string) *cobra.Command {
 
 			cfg := &config.Config{}
 
-			// Resolve the profile mirroring MustWorkspaceClient precedence:
-			// 1. --profile flag, 2. DATABRICKS_CONFIG_PROFILE env var (the SDK
-			// also reads it, but setting cfg.Profile here keeps any error
-			// messages we render referring to the same name), 3.
-			// [__settings__].default_profile in the config file.
-			if profileFlag := cmd.Flag("profile"); profileFlag != nil {
+			// Profile precedence, mirroring MustWorkspaceClient: --profile flag,
+			// then DATABRICKS_CONFIG_PROFILE (set here so our error messages name
+			// the same profile the SDK loads), then the default profile below.
+			profileFlag := cmd.Flag("profile")
+			hasProfileFlag := profileFlag != nil && profileFlag.Value.String() != ""
+			if hasProfileFlag {
 				cfg.Profile = profileFlag.Value.String()
 			}
 			if cfg.Profile == "" {
 				cfg.Profile = env.Get(cmd.Context(), "DATABRICKS_CONFIG_PROFILE")
 			}
-			if cfg.Profile == "" {
-				cfg.Profile = databrickscfg.ResolveDefaultProfile(cmd.Context())
-			}
 
-			auth.NormalizeDatabricksConfigFromEnv(cmd.Context(), cfg)
+			if hasProfileFlag {
+				// An explicit --profile wins over auth env vars; its host makes env
+				// host normalization moot (#5096).
+				cfg.Loaders = databrickscfg.ProfileAuthLoaders
+			} else {
+				auth.NormalizeDatabricksConfigFromEnv(cmd.Context(), cfg)
+			}
+			// Skips the default profile when DATABRICKS_HOST is set (#5616).
+			root.ResolveDefaultProfile(cmd.Context(), cfg)
 
 			api, err := client.New(cfg)
 			if err != nil {