source: imageblob source implementation

client/client_test.go

@@ -49,13 +49,15 @@ import (
 	"github.com/moby/buildkit/session/secrets/secretsprovider"
 	"github.com/moby/buildkit/session/sshforward/sshprovider"
 	"github.com/moby/buildkit/solver/errdefs"
+	"github.com/moby/buildkit/solver/llbsolver/provenance"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/solver/result"
 	"github.com/moby/buildkit/sourcepolicy"
 	sourcepolicypb "github.com/moby/buildkit/sourcepolicy/pb"
 	"github.com/moby/buildkit/util/attestation"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/entitlements"
+	"github.com/moby/buildkit/util/purl"
 	"github.com/moby/buildkit/util/testutil"
 	containerdutil "github.com/moby/buildkit/util/testutil/containerd"
 	"github.com/moby/buildkit/util/testutil/echoserver"
@@ -209,6 +211,7 @@ func TestIntegration(t *testing.T) {
 		testSnapshotWithMultipleBlobs,
 		testExportLocalNoPlatformSplit,
 		testExportLocalNoPlatformSplitOverwrite,
+		testImageBlobSource,
 	)
 }
 
@@ -9406,6 +9409,107 @@ func testMountStubsTimestamp(t *testing.T, sb integration.Sandbox) {
 	}
 }
 
+func testImageBlobSource(t *testing.T, sb integration.Sandbox) {
+	workers.CheckFeatureCompat(t, sb, workers.FeatureDirectPush)
+	requiresLinux(t)
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	registry, err := sb.NewRegistry()
+	if errors.Is(err, integration.ErrRequirements) {
+		t.Skip(err.Error())
+	}
+	require.NoError(t, err)
+
+	st := llb.Image("alpine")
+
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	name := registry + "/foo/blobtest:img"
+
+	_, err = c.Solve(sb.Context(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type: "image",
+				Attrs: map[string]string{
+					"name": name,
+					"push": "true",
+				},
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	desc, provider, err := contentutil.ProviderFromRef(name)
+	require.NoError(t, err)
+
+	imgs, err := testutil.ReadImages(sb.Context(), provider, desc)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(imgs.Images))
+	mfst := imgs.Images[0].Manifest
+	require.GreaterOrEqual(t, len(mfst.Layers), 1)
+
+	l := mfst.Layers[0]
+
+	blob := llb.ImageBlob(registry+"/foo/blobtest@"+l.Digest.String(), llb.Filename("layer.tar.gz"), llb.Chown(123, 456))
+	st = llb.Image("alpine").Run(llb.Shlex(`sh -c 'sha256sum /layers/layer.tar.gz | cut -d" " -f0 > /out/checksum && stat -c "%u-%g-%s" /layers/layer.tar.gz > /out/stat'`), llb.AddMount("/layers", blob, llb.Readonly)).AddMount("/out", llb.Scratch())
+
+	def, err = st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	destDir := t.TempDir()
+
+	_, err = c.Solve(sb.Context(), def, SolveOpt{
+		FrontendAttrs: map[string]string{
+			"attest:provenance": "",
+		},
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := os.ReadFile(filepath.Join(destDir, "stat"))
+	require.NoError(t, err)
+
+	require.Equal(t, "123-456-"+strconv.FormatInt(l.Size, 10), strings.TrimSpace(string(dt)))
+
+	dt, err = os.ReadFile(filepath.Join(destDir, "checksum"))
+	require.NoError(t, err)
+
+	require.Equal(t, l.Digest.Hex(), strings.TrimSpace(string(dt)))
+
+	provDt, err := os.ReadFile(filepath.Join(destDir, "provenance.json"))
+	require.NoError(t, err)
+
+	type stmtT struct {
+		Predicate provenance.ProvenancePredicate `json:"predicate"`
+	}
+	var stmt stmtT
+
+	err = json.Unmarshal(provDt, &stmt)
+	require.NoError(t, err)
+
+	expectedName, err := purl.RefToPURL("docker-blob", registry+"/foo/blobtest@"+l.Digest.String(), nil)
+	require.NoError(t, err)
+
+	found := false
+	for _, m := range stmt.Predicate.Materials {
+		if m.URI == expectedName {
+			found = true
+			require.Equal(t, m.Digest["sha256"], l.Digest.Hex())
+			break
+		}
+	}
+	require.True(t, found, "expected to find %q in %+v", expectedName, stmt.Predicate.Materials)
+}
+
 func ensureFile(t *testing.T, path string) {
 	st, err := os.Stat(path)
 	require.NoError(t, err, "expected file at %s", path)

client/llb/source.go

@@ -92,6 +92,67 @@ func (s *SourceOp) Inputs() []Output {
 	return nil
 }
 
+type ImageBlobInfo struct {
+	constraintsWrapper
+	fileinfoWrapper
+}
+
+type ImageBlobOption interface {
+	SetImageBlobOption(*ImageBlobInfo)
+}
+
+type FileInfoOption interface {
+	HTTPOption
+	ImageBlobOption
+}
+
+func ImageBlob(ref string, opts ...ImageBlobOption) State {
+	bi := &ImageBlobInfo{}
+	for _, o := range opts {
+		o.SetImageBlobOption(bi)
+	}
+	attrs := map[string]string{}
+
+	if bi.Filename != "" {
+		attrs[pb.AttrHTTPFilename] = bi.Filename
+	}
+	if bi.Perm != 0 {
+		attrs[pb.AttrHTTPPerm] = "0" + strconv.FormatInt(int64(bi.Perm), 8)
+	}
+	if bi.UID != 0 {
+		attrs[pb.AttrHTTPUID] = strconv.Itoa(bi.UID)
+	}
+	if bi.GID != 0 {
+		attrs[pb.AttrHTTPGID] = strconv.Itoa(bi.GID)
+	}
+
+	addCap(&bi.Constraints, pb.CapSourceImageBlob)
+
+	var digested reference.Digested
+
+	r, err := reference.ParseNormalizedNamed(ref)
+	if err == nil {
+		if _, tagged := r.(reference.Tagged); tagged {
+			err = errors.Errorf("tagged image reference not allowed for blob reference")
+		} else if ref, ok := r.(reference.Digested); !ok {
+			err = errors.Errorf("checksum required in blob reference")
+		} else {
+			digested = ref
+		}
+	}
+
+	repoName := "invalid"
+	if digested != nil {
+		repoName = digested.String()
+	}
+
+	source := NewSource("docker-image-blob://"+repoName, attrs, bi.Constraints)
+	if err != nil {
+		source.err = err
+	}
+	return NewState(source.Output())
+}
+
 // Image returns a state that represents a docker image in a registry.
 // Example:
 //
@@ -591,15 +652,33 @@ func HTTP(url string, opts ...HTTPOption) State {
 	return NewState(source.Output())
 }
 
-type HTTPInfo struct {
-	constraintsWrapper
-	Checksum digest.Digest
+type fileInfo struct {
 	Filename string
 	Perm     int
 	UID      int
 	GID      int
 }
 
+type fileinfoWrapper struct {
+	fileInfo
+}
+
+type fileInfoOptFunc func(f *fileInfo)
+
+func (fn fileInfoOptFunc) SetHTTPOption(hi *HTTPInfo) {
+	fn(&hi.fileInfo)
+}
+
+func (fn fileInfoOptFunc) SetImageBlobOption(ib *ImageBlobInfo) {
+	fn(&ib.fileInfo)
+}
+
+type HTTPInfo struct {
+	constraintsWrapper
+	fileinfoWrapper
+	Checksum digest.Digest
+}
+
 type HTTPOption interface {
 	SetHTTPOption(*HTTPInfo)
 }
@@ -616,22 +695,22 @@ func Checksum(dgst digest.Digest) HTTPOption {
 	})
 }
 
-func Chmod(perm os.FileMode) HTTPOption {
-	return httpOptionFunc(func(hi *HTTPInfo) {
-		hi.Perm = int(perm) & 0777
+func Chmod(perm os.FileMode) FileInfoOption {
+	return fileInfoOptFunc(func(fi *fileInfo) {
+		fi.Perm = int(perm) & 0777
 	})
 }
 
-func Filename(name string) HTTPOption {
-	return httpOptionFunc(func(hi *HTTPInfo) {
-		hi.Filename = name
+func Filename(name string) FileInfoOption {
+	return fileInfoOptFunc(func(fi *fileInfo) {
+		fi.Filename = name
 	})
 }
 
-func Chown(uid, gid int) HTTPOption {
-	return httpOptionFunc(func(hi *HTTPInfo) {
-		hi.UID = uid
-		hi.GID = gid
+func Chown(uid, gid int) FileInfoOption {
+	return fileInfoOptFunc(func(fi *fileInfo) {
+		fi.UID = uid
+		fi.GID = gid
 	})
 }
 

client/llb/state_test.go

@@ -55,6 +55,55 @@ func TestFormattingPatterns(t *testing.T) {
 	assert.Equal(t, "/foo/bar1", getDirHelper(t, s2))
 }
 
+func TestImageBlobInvalid(t *testing.T) {
+	t.Parallel()
+	ctx := context.TODO()
+
+	dgst := digest.FromBytes([]byte("foo"))
+
+	s := ImageBlob("myuser/myrepo:foo@" + string(dgst))
+	_, err := s.Marshal(ctx)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "tagged image reference not allowed")
+
+	s = ImageBlob("myuser/myrepo")
+	_, err = s.Marshal(ctx)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "checksum required in blob reference")
+
+	s = ImageBlob("myuser/myrepo@sha256:invalid")
+	_, err = s.Marshal(ctx)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "invalid reference format")
+}
+
+func TestImageBlobSource(t *testing.T) {
+	t.Parallel()
+	ctx := context.TODO()
+
+	blobDgst := digest.FromBytes([]byte("foo"))
+
+	s := ImageBlob("myuser/myrepo@" + string(blobDgst))
+	def, err := s.Marshal(ctx)
+	require.NoError(t, err)
+
+	m, arr := parseDef(t, def.Def)
+	_ = m
+	require.Equal(t, 2, len(arr))
+
+	dgst, idx := last(t, arr)
+	require.Equal(t, 0, idx)
+
+	vtx, ok := m[dgst]
+	require.Equal(t, true, ok)
+
+	src, ok := vtx.Op.(*pb.Op_Source)
+	require.Equal(t, true, ok)
+	require.Nil(t, vtx.Platform)
+
+	require.Equal(t, "docker-image-blob://docker.io/myuser/myrepo@"+string(blobDgst), src.Source.Identifier)
+}
+
 func TestStateSourceMapMarshal(t *testing.T) {
 	t.Parallel()
 

solver/llbsolver/provenance/capture.go

@@ -20,6 +20,11 @@ type ImageSource struct {
 	Local    bool
 }
 
+type ImageBlobSource struct {
+	Ref    string
+	Digest digest.Digest
+}
+
 type GitSource struct {
 	URL    string
 	Commit string
@@ -45,10 +50,11 @@ type SSH struct {
 }
 
 type Sources struct {
-	Images []ImageSource
-	Git    []GitSource
-	HTTP   []HTTPSource
-	Local  []LocalSource
+	Images     []ImageSource
+	ImageBlobs []ImageBlobSource
+	Git        []GitSource
+	HTTP       []HTTPSource
+	Local      []LocalSource
 }
 
 type Capture struct {
@@ -97,6 +103,9 @@ func (c *Capture) Sort() {
 	sort.Slice(c.Sources.Images, func(i, j int) bool {
 		return c.Sources.Images[i].Ref < c.Sources.Images[j].Ref
 	})
+	sort.Slice(c.Sources.ImageBlobs, func(i, j int) bool {
+		return c.Sources.ImageBlobs[i].Ref < c.Sources.ImageBlobs[j].Ref
+	})
 	sort.Slice(c.Sources.Local, func(i, j int) bool {
 		return c.Sources.Local[i].Name < c.Sources.Local[j].Name
 	})
@@ -161,6 +170,15 @@ func (c *Capture) AddImage(i ImageSource) {
 	c.Sources.Images = append(c.Sources.Images, i)
 }
 
+func (c *Capture) AddImageBlob(i ImageBlobSource) {
+	for _, v := range c.Sources.ImageBlobs {
+		if v.Ref == i.Ref {
+			return
+		}
+	}
+	c.Sources.ImageBlobs = append(c.Sources.ImageBlobs, i)
+}
+
 func (c *Capture) AddLocal(l LocalSource) {
 	for _, v := range c.Sources.Local {
 		if v.Name == l.Name {