diff --git a/CHANGES/9368.bugfix.rst b/CHANGES/9368.bugfix.rst
new file mode 100644
index 0000000000..7a9d8c7087
--- /dev/null
+++ b/CHANGES/9368.bugfix.rst
@@ -0,0 +1,3 @@
+Fixed proxy headers being used in the ``ConnectionKey`` hash when a proxy was not being used -- by :user:`bdraco`.
+
+If default headers are used, they are also used for proxy headers. This could have led to creating connections that were not needed when one was already available.
diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py
index f3f91ffd5b..81ab979b74 100644
--- a/aiohttp/client_reqrep.py
+++ b/aiohttp/client_reqrep.py
@@ -604,10 +604,16 @@ def update_proxy(
         proxy_auth: Optional[BasicAuth],
         proxy_headers: Optional[LooseHeaders],
     ) -> None:
+        self.proxy = proxy
+        if proxy is None:
+            self.proxy_auth = None
+            self.proxy_headers = None
+            return
+
         if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth):
             raise ValueError("proxy_auth must be None or BasicAuth() tuple")
-        self.proxy = proxy
         self.proxy_auth = proxy_auth
+
         if proxy_headers is not None and not isinstance(
             proxy_headers, (MultiDict, MultiDictProxy)
         ):
diff --git a/tests/test_client_request.py b/tests/test_client_request.py
index 7853b541fc..c9d61bf1fb 100644
--- a/tests/test_client_request.py
+++ b/tests/test_client_request.py
@@ -1467,3 +1467,30 @@ def test_basicauth_from_empty_netrc(
     """Test that no Authorization header is sent when netrc is empty"""
     req = make_request("get", "http://example.com", trust_env=True)
     assert hdrs.AUTHORIZATION not in req.headers
+
+
+async def test_connection_key_with_proxy() -> None:
+    """Verify the proxy headers are included in the ConnectionKey when a proxy is used."""
+    proxy = URL("http://proxy.example.com")
+    req = ClientRequest(
+        "GET",
+        URL("http://example.com"),
+        proxy=proxy,
+        proxy_headers={"X-Proxy": "true"},
+        loop=asyncio.get_running_loop(),
+    )
+    assert req.connection_key.proxy_headers_hash is not None
+    await req.close()
+
+
+async def test_connection_key_without_proxy() -> None:
+    """Verify the proxy headers are not included in the ConnectionKey when a proxy is used."""
+    # If proxy is unspecified, proxy_headers should be ignored
+    req = ClientRequest(
+        "GET",
+        URL("http://example.com"),
+        proxy_headers={"X-Proxy": "true"},
+        loop=asyncio.get_running_loop(),
+    )
+    assert req.connection_key.proxy_headers_hash is None
+    await req.close()