Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support offline mode #2128

Merged
merged 7 commits into from
Jan 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/input.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ type Input struct {
replaceGheActionTokenWithGithubCom string
matrix []string
actionCachePath string
actionOfflineMode bool
logPrefixJobID bool
networkName string
useNewActionCache bool
Expand Down
4 changes: 3 additions & 1 deletion cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ func Execute(ctx context.Context, version string) {
rootCmd.PersistentFlags().StringVarP(&input.cacheServerAddr, "cache-server-addr", "", common.GetOutboundIP().String(), "Defines the address to which the cache server binds.")
rootCmd.PersistentFlags().Uint16VarP(&input.cacheServerPort, "cache-server-port", "", 0, "Defines the port where the artifact server listens. 0 means a randomly available port.")
rootCmd.PersistentFlags().StringVarP(&input.actionCachePath, "action-cache-path", "", filepath.Join(CacheHomeDir, "act"), "Defines the path where the actions get cached and host workspaces created.")
rootCmd.PersistentFlags().BoolVarP(&input.actionOfflineMode, "action-offline-mode", "", false, "If action contents exists, it will not be fetch and pull again. If turn on this,will turn off force pull")
rootCmd.PersistentFlags().StringVarP(&input.networkName, "network", "", "host", "Sets a docker network name. Defaults to host.")
rootCmd.PersistentFlags().BoolVarP(&input.useNewActionCache, "use-new-action-cache", "", false, "Enable using the new Action Cache for storing Actions locally")
rootCmd.SetArgs(args())
Expand Down Expand Up @@ -582,11 +583,12 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str
EventName: eventName,
EventPath: input.EventPath(),
DefaultBranch: defaultbranch,
ForcePull: input.forcePull,
ForcePull: !input.actionOfflineMode && input.forcePull,
ForceRebuild: input.forceRebuild,
ReuseContainers: input.reuseContainers,
Workdir: input.Workdir(),
ActionCacheDir: input.actionCachePath,
ActionOfflineMode: input.actionOfflineMode,
BindWorkdir: input.bindWorkdir,
LogOutput: !input.noOutput,
JSONLogger: input.jsonLogger,
Expand Down
26 changes: 16 additions & 10 deletions pkg/common/git/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
)

if err != nil {
logger.WithError(err).Error("path", file, "not located inside a git repository")

Check warning on line 67 in pkg/common/git/git.go

View check run for this annotation

Codecov / codecov/patch

pkg/common/git/git.go#L67

Added line #L67 was not covered by tests
return "", "", err
}

Expand All @@ -74,7 +74,7 @@
}

if head.Hash().IsZero() {
return "", "", fmt.Errorf("HEAD sha1 could not be resolved")

Check warning on line 77 in pkg/common/git/git.go

View check run for this annotation

Codecov / codecov/patch

pkg/common/git/git.go#L77

Added line #L77 was not covered by tests
}

hash := head.Hash().String()
Expand Down Expand Up @@ -107,13 +107,13 @@
)

if err != nil {
return "", err
}

Check warning on line 111 in pkg/common/git/git.go

View check run for this annotation

Codecov / codecov/patch

pkg/common/git/git.go#L110-L111

Added lines #L110 - L111 were not covered by tests

iter, err := repo.References()
if err != nil {
return "", err
}

Check warning on line 116 in pkg/common/git/git.go

View check run for this annotation

Codecov / codecov/patch

pkg/common/git/git.go#L115-L116

Added lines #L115 - L116 were not covered by tests

// find the reference that matches the revision's has
err = iter.ForEach(func(r *plumbing.Reference) error {
Expand Down Expand Up @@ -146,7 +146,7 @@
})

if err != nil {
return "", err

Check warning on line 149 in pkg/common/git/git.go

View check run for this annotation

Codecov / codecov/patch

pkg/common/git/git.go#L149

Added line #L149 was not covered by tests
}

// order matters here see above comment.
Expand All @@ -157,7 +157,7 @@
return refBranch, nil
}

return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", ref)

Check warning on line 160 in pkg/common/git/git.go

View check run for this annotation

Codecov / codecov/patch

pkg/common/git/git.go#L160

Added line #L160 was not covered by tests
}

// FindGithubRepo get the repo
Expand Down Expand Up @@ -192,7 +192,7 @@
}

if len(remote.Config().URLs) < 1 {
return "", fmt.Errorf("remote '%s' exists but has no URL", remoteName)

Check warning on line 195 in pkg/common/git/git.go

View check run for this annotation

Codecov / codecov/patch

pkg/common/git/git.go#L195

Added line #L195 was not covered by tests
}

return remote.Config().URLs[0], nil
Expand Down Expand Up @@ -221,10 +221,11 @@

// NewGitCloneExecutorInput the input for the NewGitCloneExecutor
type NewGitCloneExecutorInput struct {
URL string
Ref string
Dir string
Token string
URL string
Ref string
Dir string
Token string
OfflineMode bool
}

// CloneIfRequired ...
Expand Down Expand Up @@ -302,12 +303,16 @@
return err
}

isOfflineMode := input.OfflineMode

// fetch latest changes
fetchOptions, pullOptions := gitOptions(input.Token)

err = r.Fetch(&fetchOptions)
if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
return err
if !isOfflineMode {
err = r.Fetch(&fetchOptions)
if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
return err
}

Check warning on line 315 in pkg/common/git/git.go

View check run for this annotation

Codecov / codecov/patch

pkg/common/git/git.go#L314-L315

Added lines #L314 - L315 were not covered by tests
}

var hash *plumbing.Hash
Expand Down Expand Up @@ -367,9 +372,10 @@
return err
}
}

if err = w.Pull(&pullOptions); err != nil && err != git.NoErrAlreadyUpToDate {
logger.Debugf("Unable to pull %s: %v", refName, err)
if !isOfflineMode {
if err = w.Pull(&pullOptions); err != nil && err != git.NoErrAlreadyUpToDate {
logger.Debugf("Unable to pull %s: %v", refName, err)
}

Check warning on line 378 in pkg/common/git/git.go

View check run for this annotation

Codecov / codecov/patch

pkg/common/git/git.go#L377-L378

Added lines #L377 - L378 were not covered by tests
}
logger.Debugf("Cloned %s to %s", input.URL, input.Dir)

Expand Down
9 changes: 5 additions & 4 deletions pkg/runner/reusable_workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@

remoteReusableWorkflow := newRemoteReusableWorkflow(uses)
if remoteReusableWorkflow == nil {
return common.NewErrorExecutor(fmt.Errorf("expected format {owner}/{repo}/.github/workflows/{filename}@{ref}. Actual '%s' Input string was not in a correct format", uses))
}

Check warning on line 29 in pkg/runner/reusable_workflow.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/reusable_workflow.go#L28-L29

Added lines #L28 - L29 were not covered by tests

// uses with safe filename makes the target directory look something like this {owner}-{repo}-.github-workflows-{filename}@{ref}
// instead we will just use {owner}-{repo}@{ref} as our target directory. This should also improve performance when we are using
Expand All @@ -35,8 +35,8 @@
workflowDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), safeFilename(filename))

if rc.Config.ActionCache != nil {
return newActionCacheReusableWorkflowExecutor(rc, filename, remoteReusableWorkflow)
}

Check warning on line 39 in pkg/runner/reusable_workflow.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/reusable_workflow.go#L38-L39

Added lines #L38 - L39 were not covered by tests

return common.NewPipelineExecutor(
newMutexExecutor(cloneIfRequired(rc, *remoteReusableWorkflow, workflowDir)),
Expand All @@ -44,38 +44,38 @@
)
}

func newActionCacheReusableWorkflowExecutor(rc *RunContext, filename string, remoteReusableWorkflow *remoteReusableWorkflow) common.Executor {
return func(ctx context.Context) error {
ghctx := rc.getGithubContext(ctx)
remoteReusableWorkflow.URL = ghctx.ServerURL
sha, err := rc.Config.ActionCache.Fetch(ctx, filename, remoteReusableWorkflow.CloneURL(), remoteReusableWorkflow.Ref, ghctx.Token)
if err != nil {
return err
}
archive, err := rc.Config.ActionCache.GetTarArchive(ctx, filename, sha, fmt.Sprintf(".github/workflows/%s", remoteReusableWorkflow.Filename))
if err != nil {
return err
}
defer archive.Close()
treader := tar.NewReader(archive)
if _, err = treader.Next(); err != nil {
return err
}
planner, err := model.NewSingleWorkflowPlanner(remoteReusableWorkflow.Filename, treader)
if err != nil {
return err
}
plan, err := planner.PlanEvent("workflow_call")
if err != nil {
return err
}

Check warning on line 71 in pkg/runner/reusable_workflow.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/reusable_workflow.go#L47-L71

Added lines #L47 - L71 were not covered by tests

runner, err := NewReusableWorkflowRunner(rc)
if err != nil {
return err
}

Check warning on line 76 in pkg/runner/reusable_workflow.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/reusable_workflow.go#L73-L76

Added lines #L73 - L76 were not covered by tests

return runner.NewPlanExecutor(plan)(ctx)

Check warning on line 78 in pkg/runner/reusable_workflow.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/reusable_workflow.go#L78

Added line #L78 was not covered by tests
}
}

Expand All @@ -102,10 +102,11 @@
func(ctx context.Context) error {
remoteReusableWorkflow.URL = rc.getGithubContext(ctx).ServerURL
return git.NewGitCloneExecutor(git.NewGitCloneExecutorInput{
URL: remoteReusableWorkflow.CloneURL(),
Ref: remoteReusableWorkflow.Ref,
Dir: targetDirectory,
Token: rc.Config.Token,
URL: remoteReusableWorkflow.CloneURL(),
Ref: remoteReusableWorkflow.Ref,
Dir: targetDirectory,
Token: rc.Config.Token,
OfflineMode: rc.Config.ActionOfflineMode,
})(ctx)
},
nil,
Expand All @@ -116,18 +117,18 @@
return func(ctx context.Context) error {
planner, err := model.NewWorkflowPlanner(path.Join(directory, workflow), true)
if err != nil {
return err
}

Check warning on line 121 in pkg/runner/reusable_workflow.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/reusable_workflow.go#L120-L121

Added lines #L120 - L121 were not covered by tests

plan, err := planner.PlanEvent("workflow_call")
if err != nil {
return err
}

Check warning on line 126 in pkg/runner/reusable_workflow.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/reusable_workflow.go#L125-L126

Added lines #L125 - L126 were not covered by tests

runner, err := NewReusableWorkflowRunner(rc)
if err != nil {
return err
}

Check warning on line 131 in pkg/runner/reusable_workflow.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/reusable_workflow.go#L130-L131

Added lines #L130 - L131 were not covered by tests

return runner.NewPlanExecutor(plan)(ctx)
}
Expand Down Expand Up @@ -163,8 +164,8 @@
r := regexp.MustCompile(`^([^/]+)/([^/]+)/.github/workflows/([^@]+)@(.*)$`)
matches := r.FindStringSubmatch(uses)
if len(matches) != 5 {
return nil
}

Check warning on line 168 in pkg/runner/reusable_workflow.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/reusable_workflow.go#L167-L168

Added lines #L167 - L168 were not covered by tests
return &remoteReusableWorkflow{
Org: matches[1],
Repo: matches[2],
Expand Down
1 change: 1 addition & 0 deletions pkg/runner/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
Actor string // the user that triggered the event
Workdir string // path to working directory
ActionCacheDir string // path used for caching action contents
ActionOfflineMode bool // when offline, use caching action contents
BindWorkdir bool // bind the workdir to the job container
EventName string // name of event to run
EventPath string // path to JSON file to use for event.json in containers
Expand Down Expand Up @@ -96,8 +97,8 @@
}
eventJSON, err := json.Marshal(eventMap)
if err != nil {
return nil, err
}

Check warning on line 101 in pkg/runner/runner.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/runner.go#L100-L101

Added lines #L100 - L101 were not covered by tests
runner.eventJSON = string(eventJSON)
}
return runner, nil
Expand Down Expand Up @@ -155,7 +156,7 @@

var matrixes []map[string]interface{}
if m, err := job.GetMatrixes(); err != nil {
log.Errorf("Error while get job's matrix: %v", err)

Check warning on line 159 in pkg/runner/runner.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/runner.go#L159

Added line #L159 was not covered by tests
} else {
log.Debugf("Job Matrices: %v", m)
log.Debugf("Runner Matrices: %v", runner.config.Matrix)
Expand Down Expand Up @@ -187,8 +188,8 @@
executor, err := rc.Executor()

if err != nil {
return err
}

Check warning on line 192 in pkg/runner/runner.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/runner.go#L191-L192

Added lines #L191 - L192 were not covered by tests

return executor(common.WithJobErrorContainer(WithJobLogger(ctx, rc.Run.JobID, jobName, rc.Config, &rc.Masks, matrix)))
})
Expand All @@ -197,7 +198,7 @@
}
ncpu := runtime.NumCPU()
if 1 > ncpu {
ncpu = 1

Check warning on line 201 in pkg/runner/runner.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/runner.go#L201

Added line #L201 was not covered by tests
}
log.Debugf("Detected CPUs: %d", ncpu)
return common.NewParallelExecutor(ncpu, pipeline...)(ctx)
Expand Down
9 changes: 5 additions & 4 deletions pkg/runner/step_action_remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,52 +64,53 @@
}
}
if sar.RunContext.Config.ActionCache != nil {
cache := sar.RunContext.Config.ActionCache

var err error
sar.cacheDir = fmt.Sprintf("%s/%s", sar.remoteAction.Org, sar.remoteAction.Repo)
sar.resolvedSha, err = cache.Fetch(ctx, sar.cacheDir, sar.remoteAction.URL+"/"+sar.cacheDir, sar.remoteAction.Ref, github.Token)
if err != nil {
return err
}

Check warning on line 74 in pkg/runner/step_action_remote.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/step_action_remote.go#L67-L74

Added lines #L67 - L74 were not covered by tests

remoteReader := func(ctx context.Context) actionYamlReader {
return func(filename string) (io.Reader, io.Closer, error) {
spath := filename
for i := 0; i < maxSymlinkDepth; i++ {
tars, err := cache.GetTarArchive(ctx, sar.cacheDir, sar.resolvedSha, spath)
if err != nil {
return nil, nil, os.ErrNotExist
}
treader := tar.NewReader(tars)
header, err := treader.Next()
if err != nil {
return nil, nil, os.ErrNotExist
}
if header.FileInfo().Mode()&os.ModeSymlink == os.ModeSymlink {
spath, err = symlinkJoin(spath, header.Linkname, ".")
if err != nil {
return nil, nil, err
}
} else {
return treader, tars, nil
}

Check warning on line 96 in pkg/runner/step_action_remote.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/step_action_remote.go#L76-L96

Added lines #L76 - L96 were not covered by tests
}
return nil, nil, fmt.Errorf("max depth %d of symlinks exceeded while reading %s", maxSymlinkDepth, spath)

Check warning on line 98 in pkg/runner/step_action_remote.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/step_action_remote.go#L98

Added line #L98 was not covered by tests
}
}

actionModel, err := sar.readAction(ctx, sar.Step, sar.resolvedSha, sar.remoteAction.Path, remoteReader(ctx), os.WriteFile)
sar.action = actionModel
return err

Check warning on line 104 in pkg/runner/step_action_remote.go

View check run for this annotation

Codecov / codecov/patch

pkg/runner/step_action_remote.go#L102-L104

Added lines #L102 - L104 were not covered by tests
}

actionDir := fmt.Sprintf("%s/%s", sar.RunContext.ActionCacheDir(), safeFilename(sar.Step.Uses))
gitClone := stepActionRemoteNewCloneExecutor(git.NewGitCloneExecutorInput{
URL: sar.remoteAction.CloneURL(),
Ref: sar.remoteAction.Ref,
Dir: actionDir,
Token: github.Token,
URL: sar.remoteAction.CloneURL(),
Ref: sar.remoteAction.Ref,
Dir: actionDir,
Token: github.Token,
OfflineMode: sar.RunContext.Config.ActionOfflineMode,
})
var ntErr common.Executor
if err := gitClone(ctx); err != nil {
Expand Down
Loading