fix(installer): Fixes the same pkgbase being built multiple times (Jg…

…uer#2534) * fix(installer): Fixes the same pkgbase being built multiple times When building a PKGBUILD pkgbase with multiple pkgnames, installAURPackages() invokes buildPkg() multiple times for the same pkgbase. This causes prepare() to be run multiple times for the same pkgbase, since detection of already built packages happens after prepare(). Additionally, detection of already built packages can fail if the split debug packages are enabled and the package does not contain any binaries, causing no -debug package to be created by makepkg even though it is listed by makepkg --packagelist. This commit fixes this by keeping track of the pkgdests built by buildPkg() and avoiding rebuilds of the same pkgbase in the same yay invocation. Fixes Jguer#2340. Signed-off-by: Ferdinand Bachmann <[email protected]> * fix(installer): Fixes buildPkg() isTarget param being order-dependent Previously, the buildPkg invocation for a pkgbase only considered whether the current pkgname is part of installer.origTargets. This made the decision whether to rebuild the package order-dependent. This commit fixes this by keeping track of which pkgbases are part of installer.origTargets and rebuilding the pkgbase if any of its pkgnames is part of origTargets. * fix(tests): Test that installing split packages avoids rebuilds The previous two commits changed how split packages (packages with the same pkgbase) are built, ensuring that those packages aren't built multiple times. This commit updates the lists of commands that the tests expect to be run so that `makepkg` isn't run multiple times per pkgbase. --------- Signed-off-by: Ferdinand Bachmann <[email protected]>
Ferdi265 · Nov 19, 2024 · ec837c8 · ec837c8
1 parent f100c1d
commit ec837c8
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 25 deletions.
diff --git a/local_install_test.go b/local_install_test.go
@@ -61,8 +61,6 @@ func TestIntegrationLocalInstall(t *testing.T) {
 		"pacman -D -q --asdeps --config /etc/pacman.conf -- dotnet-runtime-6.0 dotnet-sdk-6.0",
 		"makepkg --nobuild -f -C --ignorearch",
 		"makepkg -c --nobuild --noextract --ignorearch",
-		"makepkg --nobuild -f -C --ignorearch",
-		"makepkg -c --nobuild --noextract --ignorearch",
 		"pacman -U --config /etc/pacman.conf -- /testdir/jellyfin-server-10.8.4-1-x86_64.pkg.tar.zst /testdir/jellyfin-web-10.8.4-1-x86_64.pkg.tar.zst",
 		"pacman -D -q --asexplicit --config /etc/pacman.conf -- jellyfin-server jellyfin-web",
 		"makepkg --nobuild -f -C --ignorearch",
@@ -76,7 +74,6 @@ func TestIntegrationLocalInstall(t *testing.T) {
 		"git -C testdata/jfin git reset --hard HEAD",
 		"git -C testdata/jfin git merge --no-edit --ff",
 		"makepkg --packagelist",
-		"makepkg --packagelist",
 	}
 
 	captureOverride := func(cmd *exec.Cmd) (stdout string, stderr string, err error) {
@@ -331,16 +328,13 @@ func TestIntegrationLocalInstallNeeded(t *testing.T) {
 		"makepkg -c --nobuild --noextract --ignorearch",
 		"makepkg --nobuild -f -C --ignorearch",
 		"makepkg -c --nobuild --noextract --ignorearch",
-		"makepkg --nobuild -f -C --ignorearch",
-		"makepkg -c --nobuild --noextract --ignorearch",
 	}
 
 	wantCapture := []string{
 		"makepkg --packagelist",
 		"git -C testdata/jfin git reset --hard HEAD",
 		"git -C testdata/jfin git merge --no-edit --ff",
 		"makepkg --packagelist",
-		"makepkg --packagelist",
 	}
 
 	captureOverride := func(cmd *exec.Cmd) (stdout string, stderr string, err error) {
@@ -497,8 +491,6 @@ func TestIntegrationLocalInstallGenerateSRCINFO(t *testing.T) {
 		"pacman -D -q --asdeps --config /etc/pacman.conf -- dotnet-runtime-6.0 dotnet-sdk-6.0",
 		"makepkg --nobuild -f -C --ignorearch",
 		"makepkg -c --nobuild --noextract --ignorearch",
-		"makepkg --nobuild -f -C --ignorearch",
-		"makepkg -c --nobuild --noextract --ignorearch",
 		"pacman -U --config /etc/pacman.conf -- /testdir/jellyfin-server-10.8.4-1-x86_64.pkg.tar.zst /testdir/jellyfin-web-10.8.4-1-x86_64.pkg.tar.zst",
 		"pacman -D -q --asexplicit --config /etc/pacman.conf -- jellyfin-server jellyfin-web",
 		"makepkg --nobuild -f -C --ignorearch",
@@ -513,7 +505,6 @@ func TestIntegrationLocalInstallGenerateSRCINFO(t *testing.T) {
 		"git -C testdata/jfin git reset --hard HEAD",
 		"git -C testdata/jfin git merge --no-edit --ff",
 		"makepkg --packagelist",
-		"makepkg --packagelist",
 	}
 
 	captureOverride := func(cmd *exec.Cmd) (stdout string, stderr string, err error) {

diff --git a/pkg/sync/build/installer.go b/pkg/sync/build/installer.go
@@ -148,13 +148,17 @@ func (installer *Installer) handleLayer(ctx context.Context,
 	nameToBaseMap := make(map[string]string, 0)
 	syncDeps, syncExp, syncGroups := mapset.NewThreadUnsafeSet[string](),
 		mapset.NewThreadUnsafeSet[string](), mapset.NewThreadUnsafeSet[string]()
-	aurDeps, aurExp := mapset.NewThreadUnsafeSet[string](), mapset.NewThreadUnsafeSet[string]()
+	aurDeps, aurExp, aurOrigTargetBases := mapset.NewThreadUnsafeSet[string](),
+		mapset.NewThreadUnsafeSet[string](), mapset.NewThreadUnsafeSet[string]()
 
 	upgradeSync := false
 	for name, info := range layer {
 		switch info.Source {
 		case dep.AUR, dep.SrcInfo:
 			nameToBaseMap[name] = *info.AURBase
+			if installer.origTargets.Contains(name) {
+				aurOrigTargetBases.Add(*info.AURBase)
+			}
 
 			switch info.Reason {
 			case dep.Explicit:
@@ -201,14 +205,15 @@ func (installer *Installer) handleLayer(ctx context.Context,
 	}
 
 	errAur := installer.installAURPackages(ctx, cmdArgs, aurDeps, aurExp,
-		nameToBaseMap, pkgBuildDirs, true, lastLayer, installer.appendNoConfirm())
+		aurOrigTargetBases, nameToBaseMap, pkgBuildDirs, true, lastLayer,
+		installer.appendNoConfirm())
 
 	return errAur
 }
 
 func (installer *Installer) installAURPackages(ctx context.Context,
 	cmdArgs *parser.Arguments,
-	aurDepNames, aurExpNames mapset.Set[string],
+	aurDepNames, aurExpNames, aurOrigTargetBases mapset.Set[string],
 	nameToBase, pkgBuildDirsByBase map[string]string,
 	installIncompatible bool,
 	lastLayer bool,
@@ -219,23 +224,33 @@ func (installer *Installer) installAURPackages(ctx context.Context,
 		return nil
 	}
 
+	builtPkgDests := make(map[string]map[string]string)
 	deps, exps := make([]string, 0, aurDepNames.Cardinality()), make([]string, 0, aurExpNames.Cardinality())
 	pkgArchives := make([]string, 0, len(exps)+len(deps))
 
 	for _, name := range all {
 		base := nameToBase[name]
 		dir := pkgBuildDirsByBase[base]
 
-		pkgdests, errMake := installer.buildPkg(ctx, dir, base,
-			installIncompatible, cmdArgs.ExistsArg("needed"), installer.origTargets.Contains(name))
-		if errMake != nil {
-			if !lastLayer {
-				return fmt.Errorf("%s - %w", gotext.Get("error making: %s", base), errMake)
+		pkgdests, ok := builtPkgDests[base]
+		if ok {
+			installer.log.Debugln("skipping built pkgbase", base, "package", name)
+		} else {
+			var errMake error
+			installer.log.Debugln("building pkgbase", base, "package", name)
+			pkgdests, errMake = installer.buildPkg(ctx, dir, base,
+				installIncompatible, cmdArgs.ExistsArg("needed"), aurOrigTargetBases.Contains(base))
+			if errMake != nil {
+				if !lastLayer {
+					return fmt.Errorf("%s - %w", gotext.Get("error making: %s", base), errMake)
+				}
+
+				installer.failedAndIgnored[name] = errMake
+				installer.log.Errorln(gotext.Get("error making: %s", base), "-", errMake)
+				continue
 			}
 
-			installer.failedAndIgnored[name] = errMake
-			installer.log.Errorln(gotext.Get("error making: %s", base), "-", errMake)
-			continue
+			builtPkgDests[base] = pkgdests
 		}
 
 		if len(pkgdests) == 0 {

diff --git a/pkg/sync/build/installer_test.go b/pkg/sync/build/installer_test.go
@@ -696,16 +696,14 @@ func TestInstaller_InstallSplitPackage(t *testing.T) {
 				"pacman -D -q --asdeps --config /etc/pacman.conf -- dotnet-runtime-6.0 aspnet-runtime dotnet-sdk-6.0",
 				"makepkg --nobuild -f -C --ignorearch",
 				"makepkg -f -c --noconfirm --noextract --noprepare --holdver --ignorearch",
-				"makepkg --nobuild -f -C --ignorearch",
-				"makepkg -c --nobuild --noextract --ignorearch",
 				"pacman -U --config /etc/pacman.conf -- /testdir/jellyfin-web-10.8.4-1-x86_64.pkg.tar.zst /testdir/jellyfin-server-10.8.4-1-x86_64.pkg.tar.zst",
 				"pacman -D -q --asdeps --config /etc/pacman.conf -- jellyfin-server jellyfin-web",
 				"makepkg --nobuild -f -C --ignorearch",
 				"makepkg -c --nobuild --noextract --ignorearch",
 				"pacman -U --config /etc/pacman.conf -- /testdir/jellyfin-10.8.4-1-x86_64.pkg.tar.zst",
 				"pacman -D -q --asexplicit --config /etc/pacman.conf -- jellyfin",
 			},
-			wantCapture: []string{"makepkg --packagelist", "makepkg --packagelist", "makepkg --packagelist"},
+			wantCapture: []string{"makepkg --packagelist", "makepkg --packagelist"},
 		},
 	}
 

diff --git a/sync_test.go b/sync_test.go
@@ -547,7 +547,7 @@ pkgname = python-vosk
 	wantCapture := []string{
 		"/usr/bin/git -C /testdir/vosk-api reset --hard HEAD",
 		"/usr/bin/git -C /testdir/vosk-api merge --no-edit --ff",
-		"makepkg --packagelist", "makepkg --packagelist",
+		"makepkg --packagelist",
 		"makepkg --packagelist",
 	}
 	wantShow := []string{
@@ -556,7 +556,6 @@ pkgname = python-vosk
 		"makepkg -c --nobuild --noextract --ignorearch",
 		"pacman -U --config /etc/pacman.conf -- /testdir/vosk-api-0.3.45-1-x86_64.pkg.tar.zst",
 		"makepkg --nobuild -f -C --ignorearch", "makepkg -c --nobuild --noextract --ignorearch",
-		"makepkg --nobuild -f -C --ignorearch", "makepkg -c --nobuild --noextract --ignorearch",
 		"pacman -U --config /etc/pacman.conf -- /testdir/vosk-api-0.3.45-1-x86_64.pkg.tar.zst /testdir/python-vosk-0.3.45-1-x86_64.pkg.tar.zst",
 		"pacman -D -q --asdeps --config /etc/pacman.conf -- vosk-api",
 		"pacman -D -q --asexplicit --config /etc/pacman.conf -- python-vosk",