Support for namespace packages

README.rst

@@ -74,6 +74,9 @@ in development
 ^^^^^^^^^^^^^^
 
 * Drop support for Python 3.8.
+* Add support for `Namespace Packages <https://packaging.python.org/en/latest/guides/packaging-namespace-packages>`_: 
+  - Support implicit native namespace packages (PEP 420).
+  - Support legacy namespace packages as well (with `declare_namespace()` or `extend_path()`).
 
 pydoctor 24.11.2
 ^^^^^^^^^^^^^^^^

pydoctor/astbuilder.py

@@ -1206,7 +1206,21 @@ def __init__(self, system: model.System):
         self.currentMod: Optional[model.Module] = None # current module, set when visiting ast.Module.
 
         self._stack: List[model.Documentable] = []
-        self.ast_cache: Dict[Path, Optional[ast.Module]] = {}
+
+
+    def parseFile(self, path: Path, ctx: model.Module) -> Optional[ast.Module]:
+        try:
+            return self.system._ast_parser.parseFile(path)
+        except Exception as e:
+            ctx.report(f"cannot parse file, {e}")
+            return None
+
+    def parseString(self, string:str, ctx: model.Module) -> Optional[ast.Module]:
+        try:
+            return self.system._ast_parser.parseString(string)
+        except Exception:
+            ctx.report("cannot parse string")
+            return None
 
     def _push(self, 
               cls: Type[DocumentableT], 
@@ -1312,28 +1326,39 @@ def processModuleAST(self, mod_ast: ast.Module, mod: model.Module) -> None:
         vis.extensions.attach_visitor(vis)
         vis.walkabout(mod_ast)
 
-    def parseFile(self, path: Path, ctx: model.Module) -> Optional[ast.Module]:
+class SyntaxTreeParser:
+
+    def __init__(self) -> None:
+        self.ast_cache: Dict[Path, ast.Module | Exception] = {}
+
+    def parseFile(self, path: Path) -> ast.Module:
         try:
-            return self.ast_cache[path]
+            r = self.ast_cache[path]
         except KeyError:
-            mod: Optional[ast.Module] = None
+            tree: ast.Module | Exception
             try:
-                mod = parseFile(path)
-            except (SyntaxError, ValueError) as e:
-                ctx.report(f"cannot parse file, {e}")
-
-            self.ast_cache[path] = mod
-            return mod
+                tree = parseFile(path)
+                return tree
+            except Exception as e:
+                tree = e
+                raise
+            finally:
+                self.ast_cache[path] = tree
+        else:
+            if isinstance(r, Exception):
+                raise r
+            return r
 
-    def parseString(self, py_string:str, ctx: model.Module) -> Optional[ast.Module]:
+    def parseString(self, string:str) -> ast.Module:
         mod = None
         try:
-            mod = _parse(py_string)
-        except (SyntaxError, ValueError):
-            ctx.report("cannot parse string")
+            mod = _parse(string)
+        except (SyntaxError, ValueError) as e:
+            raise SyntaxError("cannot parse string") from e
         return mod
 
 model.System.defaultBuilder = ASTBuilder
+model.System.syntaxTreeParser = SyntaxTreeParser
 
 def findModuleLevelAssign(mod_ast: ast.Module) -> Iterator[Tuple[str, ast.Assign]]:
     """

pydoctor/astutils.py

@@ -737,3 +737,105 @@
 
 del _op_data, _index, _precedence_data, _symbol_data, _deprecated
 # This was part of the astor library for Python AST manipulation.
+
+
+class _OldSchoolNamespacePackageVis(ast.NodeVisitor):
+
+    is_namespace_package: bool = False
+
+    def visit_Module(self, node: ast.Module) -> None:
+        try:
+            self.generic_visit(node)
+        except StopIteration:
+            pass
+
+    def visit_skip(self, node: ast.AST) -> None:
+        pass
+
+    visit_FunctionDef = visit_AsyncFunctionDef = visit_ClassDef = visit_skip
+    visit_AugAssign = visit_skip
+    visit_Return = visit_Raise = visit_Assert = visit_skip
+    visit_Pass = visit_Break = visit_Continue = visit_Delete = visit_skip
+    visit_Global = visit_Nonlocal = visit_skip
+    visit_Import = visit_ImportFrom = visit_skip
+
+    def visit_Expr(self, node: ast.Expr) -> None:
+        # Search for ast.Expr nodes that contains a call to a name or attribute 
+        # access of "declare_namespace" and a single argument: __name__
+        if not isinstance(val:=node.value, ast.Call):
+            return
+        if not isinstance(func:=val.func, (ast.Name, ast.Attribute)):
+            return
+        if isinstance(func, ast.Name) and func.id == 'declare_namespace' or \
+           isinstance(func, ast.Attribute) and func.attr == 'declare_namespace':
+            # checks the arguments are the basic one, not custom
+            try:
+                arg1, = (*val.args, *(k.value for k in val.keywords))
+            except ValueError:
+                raise StopIteration
+            if not isinstance(arg1, ast.Name) or arg1.id != '__name__':
+                raise NamespacePackageUnsupported
+
+            self.is_namespace_package = True
+            raise StopIteration
+
+    def visit_Assign(self, node: ast.Assign) -> None:
+        # search for assignments nodes that contains a call in the 
+        # rhs to name or attribute acess of "extend_path" and two arguments: 
+        # __path__ and __name__. 
+
+        if not any(isinstance(t, ast.Name) and t.id == '__path__' for t in node.targets):
+            return
+        if not isinstance(val:=node.value, ast.Call):
+            return
+        if not isinstance(func:=val.func, (ast.Name, ast.Attribute)):
+            return
+        if isinstance(func, ast.Name) and func.id == 'extend_path' or \
+           isinstance(func, ast.Attribute) and func.attr == 'extend_path':
+            # checks the arguments are the basic one, not custom
+            try:
+                arg1, arg2 = (*val.args, *(k.value for k in val.keywords))
+            except ValueError:
+                raise StopIteration
+            if not isinstance(arg1, ast.Name) or arg1.id != '__path__':
+                raise NamespacePackageUnsupported
+            if not isinstance(arg2, ast.Name) or arg1.id != '__name__':
+                raise NamespacePackageUnsupported
+
+            self.is_namespace_package = True
+            raise StopIteration
+
+    def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
+        setattr(node, 'targets', [node.target])
+        try:
+            self.visit_Assign(node) # type:ignore[arg-type]
+        finally:
+            delattr(node, 'targets')
+
+class NamespacePackageUnsupported(Exception):
+    ...
+
+def is_old_school_namespace_package(tree: ast.Module) -> bool:
+    """
+    Returns True if the module is a pre PEP 420 namespace package::
+
+        from pkgutil import extend_path
+        __path__ = extend_path(__path__, __name__)
+        # OR
+        import pkg_resources
+        pkg_resources.declare_namespace(__name__)
+        # OR
+        __import__('pkg_resources').declare_namespace(__name__)
+        # OR
+        import pkg_resources
+        pkg_resources.declare_namespace(name=__name__)
+
+    The following code will raise an NamespacePackageUnsupported::
+
+        from pkgutil import extend_path
+        __path__ = extend_path(__path__, __name__ + '.impl')
+
+    """
+    v =_OldSchoolNamespacePackageVis()
+    v.visit(tree)
+    return v.is_namespace_package

pydoctor/epydoc2stan.py

@@ -964,6 +964,7 @@ def format_kind(kind: model.DocumentableKind, plural: bool = False) -> str:
     Transform a `model.DocumentableKind` Enum value to string.
     """
     names = {
+        model.DocumentableKind.NAMESPACE_PACKAGE : 'Namespace Package',
         model.DocumentableKind.PACKAGE         : 'Package',
         model.DocumentableKind.MODULE          : 'Module',
         model.DocumentableKind.INTERFACE       : 'Interface',