feat: add features for GitHub connections (#361)

Add features to support GitHub connections.

See SwissDataScienceCenter/renku-ui#3332

components/renku_data_services/connected_services/api.spec.yaml

@@ -186,7 +186,7 @@ paths:
             type: string
       responses:
         "200":
-          description: The connection.
+          description: The retrieved account information.
           content:
             application/json:
               schema:
@@ -195,6 +195,55 @@ paths:
           $ref: "#/components/responses/Error"
       tags:
         - oauth2
+  /oauth2/connections/{connection_id}/installations:
+    get:
+      summary: Get the installations for this OAuth2 connection for the currently authenticated user if their account is connected
+      description: This endpoint is specific to GitHub applications.
+      parameters:
+        - in: path
+          name: connection_id
+          required: true
+          schema:
+            type: string
+        - in: query
+          description: Query parameters
+          name: params
+          style: form
+          explode: true
+          schema:
+            $ref: "#/components/schemas/PaginationRequest"
+      responses:
+        "200":
+          description: The list of available GitHub installations.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/AppInstallationList"
+          headers:
+            page:
+              description: The index of the current page (starting at 1).
+              required: true
+              schema:
+                type: integer
+            per-page:
+              description: The number of items per page.
+              required: true
+              schema:
+                type: integer
+            total:
+              description: The total number of items.
+              required: true
+              schema:
+                type: integer
+            total-pages:
+              description: The total number of pages.
+              required: true
+              schema:
+                type: integer
+        default:
+          $ref: "#/components/responses/Error"
+      tags:
+        - oauth2
 components:
   schemas:
     ProviderList:
@@ -209,6 +258,8 @@ components:
           $ref: "#/components/schemas/ProviderId"
         kind:
           $ref: "#/components/schemas/ProviderKind"
+        app_slug:
+          $ref: "#/components/schemas/ApplicationSlug"
         client_id:
           $ref: "#/components/schemas/ClientId"
         client_secret:
@@ -224,6 +275,7 @@ components:
       required:
         - id
         - kind
+        - app_slug
         - client_id
         - display_name
         - scope
@@ -237,6 +289,8 @@ components:
           $ref: "#/components/schemas/ProviderId"
         kind:
           $ref: "#/components/schemas/ProviderKind"
+        app_slug:
+          $ref: "#/components/schemas/ApplicationSlug"
         client_id:
           $ref: "#/components/schemas/ClientId"
         client_secret:
@@ -262,6 +316,8 @@ components:
       properties:
         kind:
           $ref: "#/components/schemas/ProviderKind"
+        app_slug:
+          $ref: "#/components/schemas/ApplicationSlug"
         client_id:
           $ref: "#/components/schemas/ClientId"
         client_secret:
@@ -303,6 +359,33 @@ components:
       required:
         - username
         - web_url
+    AppInstallationList:
+      type: array
+      items:
+        $ref: "#/components/schemas/AppInstallation"
+    AppInstallation:
+      type: object
+      additionalProperties: false
+      properties:
+        id:
+          type: integer
+        account_login:
+          type: string
+        account_web_url:
+          type: string
+        repository_selection:
+          type: string
+          enum:
+            - all
+            - selected
+        suspended_at:
+          type: string
+          format: date-time
+      required:
+        - id
+        - account_login
+        - account_web_url
+        - repository_selection
     Ulid:
       description: ULID identifier
       type: string
@@ -319,6 +402,13 @@ components:
         - "gitlab"
         - "github"
       example: "gitlab"
+    ApplicationSlug:
+      description: |
+        URL-friendly name of the application. This field only applies to
+        GitHub Applications. The slug is provided by GitHub when
+        setting up a GitHub App.
+      type: string
+      example: "my-application"
     ClientId:
       description: |
         Client ID or Application ID value. This is provided by
@@ -360,6 +450,21 @@ components:
       description: A URL which can be opened in a browser, i.e. a web page.
       type: string
       example: "https://example.org"
+    PaginationRequest:
+      type: object
+      additionalProperties: false
+      properties:
+        page:
+          description: Result's page number starting from 1
+          type: integer
+          minimum: 1
+          default: 1
+        per_page:
+          description: The number of results per page
+          type: integer
+          minimum: 1
+          maximum: 100
+          default: 20
     ErrorResponse:
       type: object
       properties:

components/renku_data_services/connected_services/apispec.py

@@ -1,16 +1,33 @@
 # generated by datamodel-codegen:
 #   filename:  api.spec.yaml
-#   timestamp: 2024-10-18T11:06:23+00:00
+#   timestamp: 2024-08-22T08:15:52+00:00
 
 from __future__ import annotations
 
+from datetime import datetime
 from enum import Enum
 from typing import List, Optional
 
 from pydantic import ConfigDict, Field, RootModel
 from renku_data_services.connected_services.apispec_base import BaseAPISpec
 
 
+class RepositorySelection(Enum):
+    all = "all"
+    selected = "selected"
+
+
+class AppInstallation(BaseAPISpec):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    id: int
+    account_login: str
+    account_web_url: str
+    repository_selection: RepositorySelection
+    suspended_at: Optional[datetime] = None
+
+
 class ProviderKind(Enum):
     gitlab = "gitlab"
     github = "github"
@@ -21,6 +38,16 @@ class ConnectionStatus(Enum):
     pending = "pending"
 
 
+class PaginationRequest(BaseAPISpec):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    page: int = Field(1, description="Result's page number starting from 1", ge=1)
+    per_page: int = Field(
+        20, description="The number of results per page", ge=1, le=100
+    )
+
+
 class Error(BaseAPISpec):
     code: int = Field(..., example=1404, gt=0)
     detail: Optional[str] = Field(
@@ -47,6 +74,10 @@ class Oauth2ProvidersProviderIdAuthorizeGetParametersQuery(BaseAPISpec):
     authorize_params: Optional[AuthorizeParams] = None
 
 
+class Oauth2ConnectionsConnectionIdInstallationsGetParametersQuery(BaseAPISpec):
+    params: Optional[PaginationRequest] = None
+
+
 class Provider(BaseAPISpec):
     model_config = ConfigDict(
         extra="forbid",
@@ -57,6 +88,11 @@ class Provider(BaseAPISpec):
         example="some-id",
     )
     kind: ProviderKind
+    app_slug: str = Field(
+        ...,
+        description="URL-friendly name of the application. This field only applies to\nGitHub Applications. The slug is provided by GitHub when\nsetting up a GitHub App.\n",
+        example="my-application",
+    )
     client_id: str = Field(
         ...,
         description="Client ID or Application ID value. This is provided by\nthe Resource Server when setting up a new OAuth2 Client.\n",
@@ -91,6 +127,11 @@ class ProviderPost(BaseAPISpec):
         example="some-id",
     )
     kind: ProviderKind
+    app_slug: Optional[str] = Field(
+        None,
+        description="URL-friendly name of the application. This field only applies to\nGitHub Applications. The slug is provided by GitHub when\nsetting up a GitHub App.\n",
+        example="my-application",
+    )
     client_id: str = Field(
         ...,
         description="Client ID or Application ID value. This is provided by\nthe Resource Server when setting up a new OAuth2 Client.\n",
@@ -120,6 +161,11 @@ class ProviderPatch(BaseAPISpec):
         extra="forbid",
     )
     kind: Optional[ProviderKind] = None
+    app_slug: Optional[str] = Field(
+        None,
+        description="URL-friendly name of the application. This field only applies to\nGitHub Applications. The slug is provided by GitHub when\nsetting up a GitHub App.\n",
+        example="my-application",
+    )
     client_id: Optional[str] = Field(
         None,
         description="Client ID or Application ID value. This is provided by\nthe Resource Server when setting up a new OAuth2 Client.\n",
@@ -175,6 +221,10 @@ class ConnectedAccount(BaseAPISpec):
     )
 
 
+class AppInstallationList(RootModel[List[AppInstallation]]):
+    root: List[AppInstallation]
+
+
 class ProviderList(RootModel[List[Provider]]):
     root: List[Provider]
 

components/renku_data_services/connected_services/blueprints.py

@@ -1,6 +1,7 @@
 """Connected services blueprint."""
 
 from dataclasses import dataclass
+from typing import Any
 from urllib.parse import unquote, urlparse, urlunparse
 
 from sanic import HTTPResponse, Request, json, redirect
@@ -13,7 +14,8 @@
 from renku_data_services.base_api.auth import authenticate, only_admins, only_authenticated
 from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint
 from renku_data_services.base_api.misc import validate_query
-from renku_data_services.base_models.validation import validated_json
+from renku_data_services.base_api.pagination import PaginationRequest, paginate
+from renku_data_services.base_models.validation import validate_and_dump, validated_json
 from renku_data_services.connected_services import apispec
 from renku_data_services.connected_services.apispec_base import AuthorizeParams, CallbackParams
 from renku_data_services.connected_services.db import ConnectedServicesRepository
@@ -186,3 +188,26 @@ async def _get_token(_: Request, user: base_models.APIUser, connection_id: ULID)
             return json(token.dump_for_api())
 
         return "/oauth2/connections/<connection_id:ulid>/token", ["GET"], _get_token
+
+    def get_installations(self) -> BlueprintFactoryResponse:
+        """Get the installations for a specific OAuth2 connection."""
+
+        @authenticate(self.authenticator)
+        @validate_query(query=apispec.PaginationRequest)
+        @paginate
+        async def _get_installations(
+            _: Request,
+            user: base_models.APIUser,
+            connection_id: ULID,
+            pagination: PaginationRequest,
+            query: apispec.PaginationRequest,
+        ) -> tuple[list[dict[str, Any]], int]:
+            installations_list = await self.connected_services_repo.get_oauth2_app_installations(
+                connection_id=connection_id,
+                user=user,
+                pagination=pagination,
+            )
+            body = validate_and_dump(apispec.AppInstallationList, installations_list.installations)
+            return body, installations_list.total_count
+
+        return "/oauth2/connections/<connection_id:ulid>/installations", ["GET"], _get_installations

components/renku_data_services/connected_services/db.py

@@ -13,11 +13,14 @@
 from sqlalchemy.orm import selectinload
 from ulid import ULID
 
-from renku_data_services import base_models, errors
+import renku_data_services.base_models as base_models
+from renku_data_services import errors
+from renku_data_services.base_api.pagination import PaginationRequest
 from renku_data_services.connected_services import apispec, models
 from renku_data_services.connected_services import orm as schemas
-from renku_data_services.connected_services.apispec import ConnectionStatus
+from renku_data_services.connected_services.apispec import ConnectionStatus, ProviderKind
 from renku_data_services.connected_services.provider_adapters import (
+    GitHubAdapter,
     ProviderAdapter,
     get_provider_adapter,
 )
@@ -81,6 +84,7 @@ async def insert_oauth2_client(
         client = schemas.OAuth2ClientORM(
             id=provider_id,
             kind=new_client.kind,
+            app_slug=new_client.app_slug or "",
             client_id=new_client.client_id,
             client_secret=encrypted_client_secret,
             display_name=new_client.display_name,
@@ -127,7 +131,7 @@ async def update_oauth2_client(
                 client.client_secret = None
 
             for key, value in kwargs.items():
-                if key in ["kind", "client_id", "display_name", "scope", "url", "use_pkce"]:
+                if key in ["kind", "app_slug", "client_id", "display_name", "scope", "url", "use_pkce"]:
                     setattr(client, key, value)
 
             await session.flush()
@@ -326,6 +330,28 @@ async def get_oauth2_connection_token(
             token_model = models.OAuth2TokenSet.from_dict(oauth2_client.token)
             return token_model
 
+    async def get_oauth2_app_installations(
+        self, connection_id: ULID, user: base_models.APIUser, pagination: PaginationRequest
+    ) -> models.AppInstallationList:
+        """Get the installations from a OAuth2 connection."""
+        async with self.get_async_oauth2_client(connection_id=connection_id, user=user) as (
+            oauth2_client,
+            connection,
+            adapter,
+        ):
+            # NOTE: App installations are only available from GitHub
+            if connection.client.kind == ProviderKind.github and isinstance(adapter, GitHubAdapter):
+                request_url = urljoin(adapter.api_url, "user/installations")
+                params = dict(page=pagination.page, per_page=pagination.per_page)
+                response = await oauth2_client.get(request_url, params=params, headers=adapter.api_common_headers)
+
+                if response.status_code > 200:
+                    raise errors.UnauthorizedError(message="Could not get installation information.")
+
+                return adapter.api_validate_app_installations_response(response)
+
+            return models.AppInstallationList(total_count=0, installations=[])
+
     @asynccontextmanager
     async def get_async_oauth2_client(
         self, connection_id: ULID, user: base_models.APIUser