Dask Gateway unable to authenticate via JupyterHub service

[aktech]

What happened:

Dask Gateway's JupyterHub authentication is able to authenticate a JupyterHub user, but not a JupyterHub service

What you expected to happen:
Dask Gateway being able to authenticate a JupyterHub service

Minimal Complete Verifiable Example:

The easiest way to create a minimum verifiable example for this was to replicate one of the tests, if you paste the following code in this file, you'll see the test fails:

Paste here: dask-gateway/tests/test_auth.py

dask-gateway/tests/test_auth.py

Line 119 in 46cfcbf

async def test_jupyterhub_auth(monkeypatch):

@pytest.mark.skipif(not hub_mocking, reason="JupyterHub not installed")
@pytest.mark.asyncio
async def test_jupyterhub_auth_service(monkeypatch):
    from jupyterhub.tests.utils import add_user

    jhub_api_token = uuid.uuid4().hex
    jhub_service_token = uuid.uuid4().hex
    jhub_bind_url = "http://127.0.0.1:%i/@/space%%20word/" % random_port()

    hub_config = Config()
    hub_config.JupyterHub.services = [
        {"name": "dask-gateway", "api_token": jhub_api_token},
        {"name": "any-service", "api_token": jhub_service_token}
    ]
    hub_config.JupyterHub.bind_url = jhub_bind_url

    class MockHub(hub_mocking.MockHub):
        def init_logging(self):
            pass

    hub = MockHub(config=hub_config)

    # Configure gateway
    config = Config()
    config.DaskGateway.authenticator_class = (
        "dask_gateway_server.auth.JupyterHubAuthenticator"
    )
    config.JupyterHubAuthenticator.jupyterhub_api_token = jhub_api_token
    config.JupyterHubAuthenticator.jupyterhub_api_url = jhub_bind_url + "api/"

    async with temp_gateway(config=config) as g:
        async with temp_hub(hub):
            # Create a new jupyterhub user alice, and get the api token
            u = add_user(hub.db, name="alice")
            api_token = u.new_api_token()
            hub.db.commit()

            # Configure auth with incorrect api token
            auth = JupyterHubAuth(api_token=api_token)

            async with g.gateway_client(auth=auth) as gateway:
                # Auth succeeds with user token
                gateway.list_clusters()

                # Auth doesn't works with service token
                auth.api_token = jhub_api_token
                await gateway.list_clusters()

Command to run the test
Command to run this test, after pasting it in the file mentioned above (after the development environment is set):

py.test -s -k 'test_jupyterhub_auth_service'

Error:

Error Log

version.BuildInfo{Version:"v3.2.1", GitCommit:"fe51cd1e31e6a202cba7dead9552a6d418ded79a", GitTreeState:"clean", GoVersion:"go1.13.10"}
============================= test session starts ==============================
platform darwin -- Python 3.8.10, pytest-6.2.4, py-1.10.0, pluggy-0.13.1
rootdir: /Users/aktech/quansight/dask-gateway
plugins: asyncio-0.12.0
collected 116 items / 115 deselected / 5 skipped

tests/test_auth.py 12:42:37.663 [ConfigProxy] �[32minfo�[39m: Proxying http://127.0.0.1:50033 to (no default)
12:42:37.666 [ConfigProxy] �[32minfo�[39m: Proxy API at http://127.0.0.1:8001/api/routes
12:42:37.838 [ConfigProxy] �[32minfo�[39m: 200 GET /api/routes 
12:42:37.840 [ConfigProxy] �[32minfo�[39m: 200 GET /api/routes 
12:42:37.842 [ConfigProxy] �[32minfo�[39m: Adding route /@/space word -> http://127.0.0.1:8081
12:42:37.843 [ConfigProxy] �[32minfo�[39m: Route added /@/space word -> http://127.0.0.1:8081
12:42:37.843 [ConfigProxy] �[32minfo�[39m: 201 POST /api/routes/@/space%20word 
12:42:37.905 [ConfigProxy] �[33mwarn�[39m: Terminated
F

=================================== FAILURES ===================================
_________________________ test_jupyterhub_auth_service _________________________

monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7fea34547730>

    @pytest.mark.skipif(not hub_mocking, reason="JupyterHub not installed")
    @pytest.mark.asyncio
    async def test_jupyterhub_auth_service(monkeypatch):
        from jupyterhub.tests.utils import add_user
    
        jhub_api_token = uuid.uuid4().hex
        jhub_service_token = uuid.uuid4().hex
        jhub_bind_url = "http://127.0.0.1:%i/@/space%%20word/" % random_port()
    
        hub_config = Config()
        hub_config.JupyterHub.services = [
            {"name": "dask-gateway", "api_token": jhub_api_token},
            {"name": "any-service", "api_token": jhub_service_token}
        ]
        hub_config.JupyterHub.bind_url = jhub_bind_url
    
        class MockHub(hub_mocking.MockHub):
            def init_logging(self):
                pass
    
        hub = MockHub(config=hub_config)
    
        # Configure gateway
        config = Config()
        config.DaskGateway.authenticator_class = (
            "dask_gateway_server.auth.JupyterHubAuthenticator"
        )
        config.JupyterHubAuthenticator.jupyterhub_api_token = jhub_api_token
        config.JupyterHubAuthenticator.jupyterhub_api_url = jhub_bind_url + "api/"
    
        async with temp_gateway(config=config) as g:
            async with temp_hub(hub):
                # Create a new jupyterhub user alice, and get the api token
                u = add_user(hub.db, name="alice")
                api_token = u.new_api_token()
                hub.db.commit()
    
                # Configure auth with incorrect api token
                auth = JupyterHubAuth(api_token=api_token)
    
                async with g.gateway_client(auth=auth) as gateway:
                    # Auth succeeds with user token
                    gateway.list_clusters()
    
                    # Auth doesn't works with service token
                    auth.api_token = jhub_api_token
>                   await gateway.list_clusters()

tests/test_auth.py:211: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
dask-gateway/dask_gateway/client.py:434: in _clusters
    resp = await self._request("GET", url)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = Gateway<http://127.0.0.1:50035>, method = 'GET'
url = 'http://127.0.0.1:50035/api/v1/clusters/', json = None

    async def _request(self, method, url, json=None):
        if self._session is None:
            # "unsafe" allows cookies to be set for ip addresses, which can
            # commonly serve dask-gateway deployments. Since this client is
            # only ever used with a single endpoint, there is no danger of
            # leaking cookies to a different server that happens to have the
            # same ip.
            self._session = aiohttp.ClientSession(
                cookie_jar=aiohttp.CookieJar(unsafe=True)
            )
        session = self._session
    
        resp = await session.request(method, url, json=json, **self._request_kwargs)
    
        if resp.status == 401:
            headers, context = self.auth.pre_request(resp)
            resp = await session.request(
                method, url, json=json, headers=headers, **self._request_kwargs
            )
            self.auth.post_response(resp, context)
    
        if resp.status >= 400:
            try:
                msg = await resp.json()
                msg = msg["error"]
            except Exception:
                msg = await resp.text()
    
            if resp.status in {404, 422}:
                raise ValueError(msg)
            elif resp.status == 409:
                raise GatewayClusterError(msg)
            elif resp.status == 500:
>               raise GatewayServerError(msg)
E               dask_gateway.client.GatewayServerError: 500 Internal Server Error
E               
E               Server got itself in trouble

dask-gateway/dask_gateway/client.py:417: GatewayServerError
=============================== warnings summary ===============================
dask-gateway/dask_gateway/client.py:21
  /Users/aktech/quansight/dask-gateway/dask-gateway/dask_gateway/client.py:21: FutureWarning: format_bytes is deprecated and will be removed in a future release. Please use dask.utils.format_bytes instead.
    from distributed.utils import LoopRunner, format_bytes

tests/test_auth.py::test_jupyterhub_auth_service
  /Users/aktech/quansight/dask-gateway/tests/test_auth.py:207: RuntimeWarning: coroutine 'Gateway._clusters' was never awaited
    gateway.list_clusters()

-- Docs: https://docs.pytest.org/en/stable/warnings.html
=========================== short test summary info ============================
FAILED tests/test_auth.py::test_jupyterhub_auth_service - dask_gateway.client...
=========== 1 failed, 5 skipped, 115 deselected, 2 warnings in 3.53s ===========

Actual Error (missing groups key):

• JupyterHub service doesn't have a groups attribute.

[D 42:37.875 MockHub base:283] Recording first activity for <APIToken('f168...', service='dask-gateway')>
[E 2021-07-13 12:42:37.900 aiohttp.server] Error handling request
Traceback (most recent call last):
  File "/Users/aktech/anaconda3/envs/dask-gateway/lib/python3.8/site-packages/aiohttp/web_protocol.py", line 422, in _handle_request
    resp = await self._request_handler(request)
  File "/Users/aktech/anaconda3/envs/dask-gateway/lib/python3.8/site-packages/aiohttp/web_app.py", line 499, in _handle
    resp = await handler(request)
  File "/Users/aktech/quansight/dask-gateway/dask-gateway-server/dask_gateway_server/routes.py", line 13, in inner
    return await handler(request)
  File "/Users/aktech/quansight/dask-gateway/dask-gateway-server/dask_gateway_server/routes.py", line 47, in inner
    return await auth.authenticate_and_handle(request, handler)
  File "/Users/aktech/quansight/dask-gateway/dask-gateway-server/dask_gateway_server/auth.py", line 95, in authenticate_and_handle
    user = await self.authenticate(request)
  File "/Users/aktech/quansight/dask-gateway/dask-gateway-server/dask_gateway_server/auth.py", line 393, in authenticate
    return User(data["name"], groups=data["groups"], admin=data["admin"])
KeyError: 'groups'

Anything else we need to know?:

Environment:

• Dask version: source
• Python version: Python 3.8.10
• Operating System: OSX
• Install method (conda, pip, source): source