Support for VOD thumbnails (#223)

* Support for VOD thumbnails * review comments
livepeer · Oct 23, 2023 · f38d442 · f38d442
1 parent d8fa8b6
commit f38d442
Show file tree

Hide file tree

Showing 5 changed files with 118 additions and 53 deletions.
diff --git a/clients/catalyst.go b/clients/catalyst.go
@@ -59,11 +59,12 @@ type OutputLocation struct {
 }
 
 type OutputsRequest struct {
-	SourceMp4 bool   `json:"source_mp4"`
-	HLS       string `json:"hls"`
-	MP4       string `json:"mp4"`
-	FMP4      string `json:"fragmented_mp4"`
-	Clip      string `json:"clip"`
+	SourceMp4  bool   `json:"source_mp4"`
+	HLS        string `json:"hls"`
+	MP4        string `json:"mp4"`
+	FMP4       string `json:"fragmented_mp4"`
+	Clip       string `json:"clip"`
+	Thumbnails string `json:"thumbnails"`
 }
 
 type CatalystOptions struct {

diff --git a/go.mod b/go.mod
@@ -7,7 +7,7 @@ require (
 	github.com/golang/glog v1.1.2
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/livepeer/catalyst-api v0.1.2-0.20230925142340-c311569665b4
-	github.com/livepeer/go-api-client v0.4.10-0.20230926213058-6b6f4bf9bde0
+	github.com/livepeer/go-api-client v0.4.10-0.20231019205353-a945419700fc
 	github.com/livepeer/go-tools v0.3.2
 	github.com/livepeer/livepeer-data v0.7.5-0.20230927031152-b938ac1dc665
 	github.com/peterbourgon/ff v1.7.1

diff --git a/go.sum b/go.sum
@@ -281,8 +281,8 @@ github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4n
 github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo=
 github.com/livepeer/catalyst-api v0.1.2-0.20230925142340-c311569665b4 h1:UfiMdEDGa88yqYD9+i1+ldAex9Kf1+3jbq+wBrmZccM=
 github.com/livepeer/catalyst-api v0.1.2-0.20230925142340-c311569665b4/go.mod h1:Ybiub5AGDrDfvyh1MWdIa551LAwhx/6lSpbQlgb1W1Q=
-github.com/livepeer/go-api-client v0.4.10-0.20230926213058-6b6f4bf9bde0 h1:mVzmSN/dmYOtoaVhpfbDSF59quCLO6Dyvfm84z2RVqU=
-github.com/livepeer/go-api-client v0.4.10-0.20230926213058-6b6f4bf9bde0/go.mod h1:Jdb+RI7JyzEZOHd1GUuKofwFDKMO/btTa80SdpUpYQw=
+github.com/livepeer/go-api-client v0.4.10-0.20231019205353-a945419700fc h1:fN5jindUaI3PZXn6Ff8EXG8Vtvi4gxCja2fS8YI6Sw0=
+github.com/livepeer/go-api-client v0.4.10-0.20231019205353-a945419700fc/go.mod h1:Jdb+RI7JyzEZOHd1GUuKofwFDKMO/btTa80SdpUpYQw=
 github.com/livepeer/go-tools v0.3.2 h1:5pOUrOmkkGbbcWnpCt2yrSD6cD85G4GcpO4B25NpMJM=
 github.com/livepeer/go-tools v0.3.2/go.mod h1:qs31y68b3PQPmSr8nR8l5WQiIWI623z6pqOccqebjos=
 github.com/livepeer/livepeer-data v0.7.5-0.20230927031152-b938ac1dc665 h1:EXlI922Fsv9lyIw1LQ7pZN6slCuYya8NQrCFWN8INg4=

diff --git a/task/upload.go b/task/upload.go
@@ -219,12 +219,11 @@ func TaskTranscodeFile(tctx *TaskContext) (*TaskHandlerOutput, error) {
 		getOutputLocations: func() ([]clients.OutputLocation, error) {
 			_, outputLocation, err := outputLocations(
 				params.Storage.URL,
-				isEnabled(params.Outputs.HLS.Path),
-				params.Outputs.HLS.Path,
-				isEnabled(params.Outputs.MP4.Path),
-				params.Outputs.MP4.Path,
-				isEnabled(params.Outputs.FMP4.Path),
-				params.Outputs.FMP4.Path,
+				outputs{
+					hls:  out(isEnabled(params.Outputs.HLS.Path), params.Outputs.HLS.Path),
+					mp4:  out(isEnabled(params.Outputs.MP4.Path), params.Outputs.MP4.Path),
+					fmp4: out(isEnabled(params.Outputs.FMP4.Path), params.Outputs.FMP4.Path),
+				},
 				false,
 			)
 			return outputLocation, err
@@ -677,7 +676,19 @@ func uploadTaskOutputLocations(tctx *TaskContext) ([]OutputName, []clients.Outpu
 	} else {
 		mp4 = OUTPUT_ONLY_SHORT
 	}
-	outputNames, outputLocations, err := outputLocations(outURL, OUTPUT_ENABLED, playbackId, mp4, playbackId, "", "", !isEncryptionEnabled(*tctx.Task.Params.Upload))
+	thumbsEnabled := ""
+	if tctx.Task.Params.Upload.Thumbnails {
+		thumbsEnabled = OUTPUT_ENABLED
+	}
+	outputNames, outputLocations, err := outputLocations(
+		outURL,
+		outputs{
+			hls:        out(OUTPUT_ENABLED, playbackId),
+			mp4:        out(mp4, playbackId),
+			thumbnails: out(thumbsEnabled, playbackId),
+		},
+		!isEncryptionEnabled(*tctx.Task.Params.Upload),
+	)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -718,7 +729,14 @@ func clipTaskOutputLocations(tctx *TaskContext) ([]OutputName, []clients.OutputL
 		return nil, nil, fmt.Errorf("error parsing object store URL %w", err)
 	}
 
-	outputNames, outputLocations, err := outputLocations(outURL, OUTPUT_ENABLED, playbackId, OUTPUT_ENABLED, playbackId, "", "", false)
+	outputNames, outputLocations, err := outputLocations(
+		outURL,
+		outputs{
+			hls: out(OUTPUT_ENABLED, playbackId),
+			mp4: out(OUTPUT_ENABLED, playbackId),
+		},
+		false,
+	)
 
 	if err != nil {
 		return nil, nil, err
@@ -738,14 +756,30 @@ func clipTaskOutputLocations(tctx *TaskContext) ([]OutputName, []clients.OutputL
 	return outputNames, outputLocations, nil
 }
 
+// out is a helper to create an output, to avoid repeated struct attribute names in the code
+func out(enabled, path string) output {
+	return output{
+		enabled: enabled,
+		path:    path,
+	}
+}
+
+// output represents each requested media output for outputLocations()
+type output struct {
+	enabled string
+	path    string
+}
+
+type outputs struct {
+	hls        output
+	mp4        output
+	fmp4       output
+	thumbnails output
+}
+
 func outputLocations(
-	outURL,
-	hls,
-	hlsRelPath,
-	mp4,
-	mp4RelPath,
-	fmp4,
-	fmp4RelPath string,
+	outURL string,
+	outs outputs,
 	sourceCopy bool,
 ) ([]OutputName, []clients.OutputLocation, error) {
 	url, err := url.Parse(outURL)
@@ -755,46 +789,35 @@ func outputLocations(
 	names, locations :=
 		[]OutputName{OutputNameOSPlaylistHLS, OutputNameEmpty},
 		[]clients.OutputLocation{
-			{
-				Type: "object_store",
-				URL:  url.JoinPath(hlsRelPath).String(),
-				Outputs: &clients.OutputsRequest{
-					HLS: hls,
-				},
-			},
-			{
-				Type: "object_store",
-				URL:  url.JoinPath(mp4RelPath).String(),
-				Outputs: &clients.OutputsRequest{
-					MP4: mp4,
-				},
-			},
+			outputLocation(url, outs.hls.path, &clients.OutputsRequest{HLS: outs.hls.enabled}),
+			outputLocation(url, outs.mp4.path, &clients.OutputsRequest{MP4: outs.mp4.enabled}),
 		}
-	if fmp4 == OUTPUT_ENABLED {
+	if outs.fmp4.enabled == OUTPUT_ENABLED {
 		names, locations =
 			append(names, OutputNameEmpty),
-			append(locations, clients.OutputLocation{
-				Type: "object_store",
-				URL:  url.JoinPath(fmp4RelPath).String(),
-				Outputs: &clients.OutputsRequest{
-					FMP4: fmp4,
-				},
-			})
+			append(locations, outputLocation(url, outs.fmp4.path, &clients.OutputsRequest{FMP4: outs.fmp4.enabled}))
 	}
 	if sourceCopy {
 		names, locations =
 			append(names, OutputNameOSSourceMP4),
-			append(locations, clients.OutputLocation{
-				Type: "object_store",
-				URL:  url.JoinPath(videoFileName(hlsRelPath)).String(),
-				Outputs: &clients.OutputsRequest{
-					SourceMp4: true,
-				},
-			})
+			append(locations, outputLocation(url, videoFileName(outs.hls.path), &clients.OutputsRequest{SourceMp4: true}))
+	}
+	if outs.thumbnails.enabled == OUTPUT_ENABLED {
+		names, locations =
+			append(names, OutputNameEmpty),
+			append(locations, outputLocation(url, outs.thumbnails.path, &clients.OutputsRequest{Thumbnails: outs.thumbnails.enabled}))
 	}
 	return names, locations, nil
 }
 
+func outputLocation(url *url.URL, path string, request *clients.OutputsRequest) clients.OutputLocation {
+	return clients.OutputLocation{
+		Type:    "object_store",
+		URL:     url.JoinPath(path).String(),
+		Outputs: request,
+	}
+}
+
 func catalystTaskAttemptID(task *api.Task) string {
 	// Simplest way to identify unique runs of a given task. We should think of
 	// something more sophisticated in the future.

diff --git a/task/upload_test.go b/task/upload_test.go
@@ -160,6 +160,8 @@ func TestOutputLocations(t *testing.T) {
 		hlsRelPath              string
 		mp4                     string
 		mp4RelPath              string
+		thumbs                  string
+		thumbsRelPath           string
 		sourceCopy              bool
 		expectedOutputLocations []clients.OutputLocation
 		hasError                bool
@@ -244,10 +246,49 @@ func TestOutputLocations(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:          "Thumbnails",
+			outURL:        "s3+https://user:[email protected]/outbucket",
+			hls:           "enabled",
+			hlsRelPath:    "video/hls",
+			thumbs:        "enabled",
+			thumbsRelPath: "video/thumbs",
+			expectedOutputLocations: []clients.OutputLocation{
+				{
+					Type: "object_store",
+					URL:  "s3+https://user:[email protected]/outbucket/video/hls",
+					Outputs: &clients.OutputsRequest{
+						HLS: "enabled",
+					},
+				},
+				{
+					Type: "object_store",
+					URL:  "s3+https://user:[email protected]/outbucket",
+					Outputs: &clients.OutputsRequest{
+						MP4: "",
+					},
+				},
+				{
+					Type: "object_store",
+					URL:  "s3+https://user:[email protected]/outbucket/video/thumbs",
+					Outputs: &clients.OutputsRequest{
+						Thumbnails: "enabled",
+					},
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, gotOutputLocations, err := outputLocations(tt.outURL, tt.hls, tt.hlsRelPath, tt.mp4, tt.mp4RelPath, "", "", tt.sourceCopy)
+			_, gotOutputLocations, err := outputLocations(
+				tt.outURL,
+				outputs{
+					hls:        out(tt.hls, tt.hlsRelPath),
+					mp4:        out(tt.mp4, tt.mp4RelPath),
+					thumbnails: out(tt.thumbs, tt.thumbsRelPath),
+				},
+				tt.sourceCopy,
+			)
 
 			if tt.hasError {
 				require.Error(t, err)