feat: Generate credits when building docs

We the macros plugin to inject credits
into the documentation at build time.
We also reduced the time needed to generate
credits by using asynchronous requests
to pypi.python.org.

project/docs/credits.md

@@ -1 +1 @@
-{!CREDITS.md!}
+{{ credits() }}

project/docs/macros.py

@@ -0,0 +1,72 @@
+import asyncio
+import os
+import re
+from itertools import chain
+from pathlib import Path
+from shutil import which
+from typing import List, Optional, Pattern
+
+import httpx
+import toml
+from duty import duty
+from git_changelog.build import Changelog, Version
+from jinja2 import StrictUndefined
+from jinja2.sandbox import SandboxedEnvironment
+from pip._internal.commands.show import search_packages_info  # noqa: WPS436 (no other way?)
+
+
+def get_credits_data() -> dict:
+    """
+    Return data used to generate the credits file.
+
+    Returns:
+        Data required to render the credits template.
+    """
+    project_dir = Path(__file__).parent.parent
+    metadata = toml.load(project_dir / "pyproject.toml")["tool"]["poetry"]
+    lock_data = toml.load(project_dir / "poetry.lock")
+    project_name = metadata["name"]
+
+    poetry_dependencies = chain(metadata["dependencies"].keys(), metadata["dev-dependencies"].keys())
+    direct_dependencies = {dep.lower() for dep in poetry_dependencies}
+    direct_dependencies.remove("python")
+    indirect_dependencies = {pkg["name"].lower() for pkg in lock_data["package"]}
+    indirect_dependencies -= direct_dependencies
+    dependencies = direct_dependencies | indirect_dependencies
+
+    packages = {}
+    for pkg in search_packages_info(dependencies):
+        pkg = {_: pkg[_] for _ in ("name", "home-page")}
+        packages[pkg["name"].lower()] = pkg
+
+    loop = asyncio.get_event_loop()
+    client = httpx.AsyncClient()
+    to_get = [dep for dep in dependencies if dep not in packages]
+    coroutines = [client.get(f"https://pypi.python.org/pypi/{dep}/json") for dep in to_get]
+    responses = loop.run_until_complete(asyncio.gather(*coroutines))
+    loop.run_until_complete(client.aclose())
+
+    for response in responses:
+        pkg_data = response.json()["info"]
+        home_page = pkg_data["home_page"] or pkg_data["project_url"] or pkg_data["package_url"]
+        pkg_name = pkg_data["name"]
+        package = {"name": pkg_name, "home-page": home_page}
+        packages.update({pkg_name.lower(): package})
+
+    return {
+        "project_name": project_name,
+        "direct_dependencies": sorted(direct_dependencies),
+        "indirect_dependencies": sorted(indirect_dependencies),
+        "package_info": packages,
+    }
+
+
+def define_env(env):
+    @env.macro
+    def credits():
+        template_url = "https://raw.githubusercontent.com/pawamoy/jinja-templates/master/credits.md"
+        env = SandboxedEnvironment(undefined=StrictUndefined)
+        template_data = get_credits_data()
+        template_text = httpx.get(template_url).text
+        rendered = env.from_string(template_text).render(**template_data)
+        return rendered

project/duties.py

@@ -2,18 +2,14 @@
 
 import os
 import re
-from itertools import chain
 from pathlib import Path
 from shutil import which
 from typing import List, Optional, Pattern
 
 import httpx
-import toml
 from duty import duty
 from git_changelog.build import Changelog, Version
-from jinja2 import StrictUndefined
 from jinja2.sandbox import SandboxedEnvironment
-from pip._internal.commands.show import search_packages_info  # noqa: WPS436 (no other way?)
 
 PY_SRC_PATHS = (Path(_) for _ in ("src", "tests", "duties.py"))
 PY_SRC_LIST = tuple(str(_) for _ in PY_SRC_PATHS)
@@ -233,78 +229,7 @@ def clean(ctx):
     ctx.run("find . -name '*.rej' -delete")
 
 
-def get_credits_data() -> dict:
-    """
-    Return data used to generate the credits file.
-
-    Returns:
-        Data required to render the credits template.
-    """
-    project_dir = Path(__file__).parent.parent
-    metadata = toml.load(project_dir / "pyproject.toml")["tool"]["poetry"]
-    lock_data = toml.load(project_dir / "poetry.lock")
-    project_name = metadata["name"]
-
-    poetry_dependencies = chain(metadata["dependencies"].keys(), metadata["dev-dependencies"].keys())
-    direct_dependencies = {dep.lower() for dep in poetry_dependencies}
-    direct_dependencies.remove("python")
-    indirect_dependencies = {pkg["name"].lower() for pkg in lock_data["package"]}
-    indirect_dependencies -= direct_dependencies
-    dependencies = direct_dependencies | indirect_dependencies
-
-    packages = {}
-    for pkg in search_packages_info(dependencies):
-        pkg = {_: pkg[_] for _ in ("name", "home-page")}
-        packages[pkg["name"].lower()] = pkg
-
-    for dependency in dependencies:
-        if dependency not in packages:
-            pkg_data = httpx.get(f"https://pypi.python.org/pypi/{dependency}/json").json()["info"]
-            home_page = pkg_data["home_page"] or pkg_data["project_url"] or pkg_data["package_url"]
-            pkg_name = pkg_data["name"]
-            package = {"name": pkg_name, "home-page": home_page}
-            packages.update({pkg_name.lower(): package})
-
-    return {
-        "project_name": project_name,
-        "direct_dependencies": sorted(direct_dependencies),
-        "indirect_dependencies": sorted(indirect_dependencies),
-        "package_info": packages,
-    }
-
-
 @duty
-def docs_regen(ctx):
-    """
-    Regenerate some documentation pages.
-
-    Arguments:
-        ctx: The context instance (passed automatically).
-    """
-    url_prefix = "https://raw.githubusercontent.com/pawamoy/jinja-templates/master/"
-    regen_list = (("CREDITS.md", get_credits_data, url_prefix + "credits.md"),)
-
-    def regen() -> int:  # noqa: WPS430 (nested function)
-        """
-        Regenerate pages listed in global `REGEN` list.
-
-        Returns:
-            An exit code.
-        """
-        env = SandboxedEnvironment(undefined=StrictUndefined)
-        for target, get_data, template in regen_list:
-            print("Regenerating", target)  # noqa: WPS421 (print)
-            template_data = get_data()
-            template_text = httpx.get(template).text
-            rendered = env.from_string(template_text).render(**template_data)
-            with open(target, "w") as stream:
-                stream.write(rendered)
-        return 0
-
-    ctx.run(regen, title="Regenerating docfiles", pty=PTY)
-
-
-@duty(pre=[docs_regen])
 def docs(ctx):
     """
     Build the documentation locally.
@@ -315,7 +240,7 @@ def docs(ctx):
     ctx.run("mkdocs build", title="Building documentation")
 
 
-@duty(pre=[docs_regen])
+@duty
 def docs_serve(ctx, host="127.0.0.1", port=8000):
     """
     Serve the documentation (localhost:8000).
@@ -328,7 +253,7 @@ def docs_serve(ctx, host="127.0.0.1", port=8000):
     ctx.run(f"mkdocs serve -a {host}:{port}", title="Serving documentation", capture=False)
 
 
-@duty(pre=[docs_regen])
+@duty
 def docs_deploy(ctx):
     """
     Deploy the documentation on GitHub pages.

project/mkdocs.yml.jinja

@@ -6,15 +6,15 @@ repo_name: "[[ repository_namespace ]]/[[ repository_name ]]"
 
 nav:
 - Home:
-- Overview: index.md
+  - Overview: index.md
   - Changelog: changelog.md
   - Credits: credits.md
   - License: license.md
 - Code Reference:
   - cli.py: reference/cli.md
 - Development:
-- Contributing: contributing.md
-- Code of Conduct: code_of_conduct.md
+  - Contributing: contributing.md
+  - Code of Conduct: code_of_conduct.md
   - Coverage report: coverage.md
 
 theme:
@@ -48,3 +48,5 @@ plugins:
 - mkdocstrings:
     watch:
     - src/[[ python_package_import_name ]]
+- macros:
+    module_name: docs/macros

project/pyproject.toml.jinja

@@ -39,6 +39,7 @@ isort = {version = "<5", extras = ["pyproject"]}
 jinja2-cli = "^0.7.0"
 mkdocs = "^1.1.2"
 mkdocs-coverage = "^0.2.1"
+mkdocs-macros-plugin = "^0.5.0"
 mkdocs-material = "^6.2.7"
 mkdocstrings = "^0.14.0"
 mypy = "^0.800"

tests/test_generate.sh

@@ -53,14 +53,16 @@ echo
 echo ">>> Running initial quality checks"
 make --no-print-directory check
 echo
-echo ">>> Generating docs, formatting, and re-running quality checks"
-make --no-print-directory docs-regen format check-code-quality
+echo ">>> Formatting, and re-running quality checks"
+make --no-print-directory format check-code-quality
 echo
 echo ">>> Running tests"
 make --no-print-directory test
 echo
 echo ">>> Creating second commit (fix)"
-git commit -am "fix: Fix all bugs"
+touch empty
+git add empty
+git commit -m "fix: Fix all bugs"
 echo
 echo ">>> Updating changelog and releasing version"
 make --no-print-directory changelog release version=0.1.1