From d52f8ee34d6c34beef36d02a4f476f38dd3cf1ec Mon Sep 17 00:00:00 2001 From: Doug MacEachern Date: Thu, 12 Oct 2023 15:00:11 -0700 Subject: [PATCH] api: add Content Library update session file APIs Closes #3258 --- govc/USAGE.md | 3 + govc/library/import.go | 5 + govc/library/probe.go | 115 ++++++++++++++++++ govc/library/session/ls.go | 52 ++++++-- govc/library/session/rm.go | 18 ++- govc/test/library.bats | 56 +++++++++ .../library_item_updatesession_file.go | 111 +++++++++++------ vapi/simulator/simulator.go | 95 ++++++++++++++- 8 files changed, 400 insertions(+), 55 deletions(-) create mode 100644 govc/library/probe.go mode change 100755 => 100644 vapi/simulator/simulator.go diff --git a/govc/USAGE.md b/govc/USAGE.md index 9fca109b9..a6c4648fd 100644 --- a/govc/USAGE.md +++ b/govc/USAGE.md @@ -3476,6 +3476,7 @@ Examples: govc library.session.ls -json | jq . Options: + -i=false List session item files (with -json only) ``` ## library.session.rm @@ -3487,9 +3488,11 @@ Remove a library item update session. Examples: govc library.session.rm session_id + govc library.session.rm -i session_id foo.ovf Options: -f=false Cancel session if active + -i=false Remove session item file ``` ## library.subscriber.create diff --git a/govc/library/import.go b/govc/library/import.go index 7304bf218..17236bc7d 100644 --- a/govc/library/import.go +++ b/govc/library/import.go @@ -177,6 +177,11 @@ func (cmd *item) Run(ctx context.Context, f *flag.FlagSet) error { return err } + err = m.CompleteLibraryItemUpdateSession(ctx, session) + if err != nil { + return err + } + return m.WaitOnLibraryItemUpdateSession(ctx, session, 3*time.Second, nil) } diff --git a/govc/library/probe.go b/govc/library/probe.go new file mode 100644 index 000000000..afdbd791e --- /dev/null +++ b/govc/library/probe.go @@ -0,0 +1,115 @@ +/* +Copyright (c) 2023-2023 VMware, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package library + +import ( + "context" + "flag" + "fmt" + "io" + "os" + "text/tabwriter" + + "github.com/vmware/govmomi/govc/cli" + "github.com/vmware/govmomi/govc/flags" + "github.com/vmware/govmomi/vapi/library" +) + +type probe struct { + *flags.ClientFlag + *flags.OutputFlag + + fail bool +} + +func init() { + cli.Register("library.probe", &probe{}, true) +} + +func (cmd *probe) Register(ctx context.Context, f *flag.FlagSet) { + cmd.ClientFlag, ctx = flags.NewClientFlag(ctx) + cmd.ClientFlag.Register(ctx, f) + + cmd.OutputFlag, ctx = flags.NewOutputFlag(ctx) + cmd.OutputFlag.Register(ctx, f) + + f.BoolVar(&cmd.fail, "f", false, "Fail if probe status is not success") +} + +func (cmd *probe) Usage() string { + return "URI" +} + +func (cmd *probe) Description() string { + return `Probes the source endpoint URI with https or http schemes. + +Examples: + govc library.probe https://example.com/file.ova` +} + +type probeResult struct { + *library.ProbeResult +} + +func (r *probeResult) Write(w io.Writer) error { + tw := tabwriter.NewWriter(w, 2, 0, 2, ' ', 0) + + fmt.Fprintf(tw, "Status:\t%s\n", r.Status) + thumbprint := r.SSLThumbprint + if thumbprint == "" { + thumbprint = "-" + } + fmt.Fprintf(tw, "Thumbprint:\t%s\n", thumbprint) + for _, e := range r.ErrorMessages { + fmt.Fprintf(tw, "%s:\t%s\n", e.ID, e.Error()) + } + + return tw.Flush() +} + +func (cmd *probe) Process(ctx context.Context) error { + if err := cmd.ClientFlag.Process(ctx); err != nil { + return err + } + return cmd.OutputFlag.Process(ctx) +} + +func (cmd *probe) Run(ctx context.Context, f *flag.FlagSet) error { + if f.NArg() != 1 { + return flag.ErrHelp + } + + c, err := cmd.RestClient() + if err != nil { + return err + } + + m := library.NewManager(c) + + p, err := m.ProbeTransferEndpoint(ctx, library.TransferEndpoint{URI: f.Arg(0)}) + if err != nil { + return err + } + + if cmd.fail && p.Status != "SUCCESS" { + cmd.Out = os.Stderr + // using same exit code as curl -f: + defer os.Exit(22) + } + + return cmd.WriteResult(&probeResult{p}) +} diff --git a/govc/library/session/ls.go b/govc/library/session/ls.go index 99872ca33..abd90d5d3 100644 --- a/govc/library/session/ls.go +++ b/govc/library/session/ls.go @@ -31,6 +31,8 @@ import ( type ls struct { *flags.ClientFlag *flags.OutputFlag + + files bool } func init() { @@ -42,6 +44,8 @@ func (cmd *ls) Register(ctx context.Context, f *flag.FlagSet) { cmd.ClientFlag.Register(ctx, f) cmd.OutputFlag, ctx = flags.NewOutputFlag(ctx) cmd.OutputFlag.Register(ctx, f) + + f.BoolVar(&cmd.files, "i", false, "List session item files (with -json only)") } func (cmd *ls) Process(ctx context.Context) error { @@ -59,8 +63,14 @@ Examples: govc library.session.ls -json | jq .` } +type librarySession struct { + *library.Session + LibraryItemPath string `json:"library_item_path"` +} + type info struct { - Sessions []*library.Session `json:"sessions"` + Sessions []librarySession `json:"sessions"` + Files map[string]any `json:"files"` kind string } @@ -70,7 +80,7 @@ func (i *info) Write(w io.Writer) error { for _, s := range i.Sessions { _, _ = fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%d\t%s\t%s\n", - s.ID, s.LibraryItemID, i.kind, s.LibraryItemContentVersion, s.ClientProgress, s.State, + s.ID, s.LibraryItemPath, i.kind, s.LibraryItemContentVersion, s.ClientProgress, s.State, s.ExpirationTime.Format("2006-01-02 15:04")) } @@ -102,26 +112,46 @@ func (cmd *ls) Run(ctx context.Context, f *flag.FlagSet) error { if len(ids) == 0 { continue } - var sessions []*library.Session + i := &info{ + Files: make(map[string]any), + kind: k.kind, + } for _, id := range ids { session, err := k.get(ctx, id) if err != nil { return err } + var path string item, err := m.GetLibraryItem(ctx, session.LibraryItemID) - if err != nil { - return err + if err == nil { + // can only show library path if item exists + lib, err := m.GetLibraryByID(ctx, item.LibraryID) + if err != nil { + return err + } + path = fmt.Sprintf("/%s/%s", lib.Name, item.Name) } - lib, err := m.GetLibraryByID(ctx, item.LibraryID) - if err != nil { - return err + i.Sessions = append(i.Sessions, librarySession{session, path}) + if !cmd.files { + continue + } + if k.kind == "Update" { + f, err := m.ListLibraryItemUpdateSessionFile(ctx, id) + if err != nil { + return err + } + i.Files[id] = f + } else { + f, err := m.ListLibraryItemDownloadSessionFile(ctx, id) + if err != nil { + return err + } + i.Files[id] = f } - session.LibraryItemID = fmt.Sprintf("/%s/%s", lib.Name, item.Name) - sessions = append(sessions, session) } - err = cmd.WriteResult(&info{sessions, k.kind}) + err = cmd.WriteResult(i) if err != nil { return err } diff --git a/govc/library/session/rm.go b/govc/library/session/rm.go index d1b30623f..68efee3d7 100644 --- a/govc/library/session/rm.go +++ b/govc/library/session/rm.go @@ -1,11 +1,11 @@ /* -Copyright (c) 2019 VMware, Inc. All Rights Reserved. +Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -18,6 +18,7 @@ package session import ( "context" + "errors" "flag" "github.com/vmware/govmomi/govc/cli" @@ -29,6 +30,7 @@ type rm struct { *flags.ClientFlag cancel bool + files bool } func init() { @@ -40,13 +42,15 @@ func (cmd *rm) Register(ctx context.Context, f *flag.FlagSet) { cmd.ClientFlag.Register(ctx, f) f.BoolVar(&cmd.cancel, "f", false, "Cancel session if active") + f.BoolVar(&cmd.files, "i", false, "Remove session item file") } func (cmd *rm) Description() string { return `Remove a library item update session. Examples: - govc library.session.rm session_id` + govc library.session.rm session_id + govc library.session.rm -i session_id foo.ovf` } func (cmd *rm) Run(ctx context.Context, f *flag.FlagSet) error { @@ -60,11 +64,19 @@ func (cmd *rm) Run(ctx context.Context, f *flag.FlagSet) error { m := library.NewManager(c) cancel := m.CancelLibraryItemUpdateSession remove := m.DeleteLibraryItemUpdateSession + rmfile := m.RemoveLibraryItemUpdateSessionFile _, err = m.GetLibraryItemUpdateSession(ctx, id) if err != nil { cancel = m.CancelLibraryItemDownloadSession remove = m.DeleteLibraryItemDownloadSession + rmfile = func(context.Context, string, string) error { + return errors.New("cannot delete a download session file") + } + } + + if cmd.files { + return rmfile(ctx, id, f.Arg(1)) } if cmd.cancel { diff --git a/govc/test/library.bats b/govc/test/library.bats index 69f69d2ad..cdb064d7d 100755 --- a/govc/test/library.bats +++ b/govc/test/library.bats @@ -551,3 +551,59 @@ EOF # remove generated cert and key rm "$pem".{crt,key} } + +@test "library.session" { + vcsim_env + + run govc library.session.ls + assert_success + + run govc library.create my-content + assert_success + + run govc library.import /my-content "$GOVC_IMAGES/$TTYLINUX_NAME.ova" + assert_success + + run govc library.session.ls + assert_success + assert_matches ttylinux + + run govc library.session.ls -json + assert_success + + run govc library.session.ls -json -i + assert_success + + n=$(govc library.session.ls -json -i | jq '.files[] | length') + assert_equal 2 "$n" # .ovf + .vmdk + + id=$(govc library.session.ls -json | jq -r .sessions[].id) + + run govc library.session.rm -i "$id" ttylinux-pc_i486-16.1.ovf + assert_failure # removeFile not allowed in state DONE + assert_matches "500 Internal Server Error" + + run govc library.session.rm "$id" + assert_success +} + +@test "library.probe" { + vcsim_env + + export GOVC_SHOW_UNRELEASED=true + + run govc library.probe + assert_failure + + run govc library.probe https://www.vmware.com + assert_success + + run govc library.probe -f ftp://www.vmware.com + if [ "$status" -ne 22 ]; then + flunk $(printf "expected failed exit status=22, got status=%d" $status) + fi + + run govc library.probe -json ftp://www.vmware.com + assert_success + assert_matches INVALID_URL +} diff --git a/vapi/library/library_item_updatesession_file.go b/vapi/library/library_item_updatesession_file.go index e34331cfa..c571d3273 100644 --- a/vapi/library/library_item_updatesession_file.go +++ b/vapi/library/library_item_updatesession_file.go @@ -1,11 +1,11 @@ /* -Copyright (c) 2018 VMware, Inc. All Rights Reserved. +Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -34,6 +34,13 @@ type TransferEndpoint struct { SSLCertificateThumbprint string `json:"ssl_certificate_thumbprint,omitempty"` } +type ProbeResult struct { + Status string `json:"status"` + SSLThumbprint string `json:"ssl_thumbprint,omitempty"` + SSLCertificate string `json:"ssl_certificate,omitempty"` + ErrorMessages []rest.LocalizableMessage `json:"error_messages,omitempty"` +} + // UpdateFile is the specification for the updatesession // operations file:add, file:get, and file:list. type UpdateFile struct { @@ -48,6 +55,19 @@ type UpdateFile struct { UploadEndpoint *TransferEndpoint `json:"upload_endpoint,omitempty"` } +// FileValidationError contains the validation error of a file in the update session +type FileValidationError struct { + Name string `json:"name"` + ErrorMessage rest.LocalizableMessage `json:"error_message"` +} + +// UpdateFileValidation contains the result of validating the files in the update session +type UpdateFileValidation struct { + HasErrors bool `json:"has_errors"` + MissingFiles []string `json:"missing_files,omitempty"` + InvalidFiles []FileValidationError `json:"invalid_files,omitempty"` +} + // AddLibraryItemFile adds a file func (c *Manager) AddLibraryItemFile(ctx context.Context, sessionID string, updateFile UpdateFile) (*UpdateFile, error) { url := c.Resource(internal.LibraryItemUpdateSessionFile).WithID(sessionID).WithAction("add") @@ -66,29 +86,31 @@ func (c *Manager) AddLibraryItemFile(ctx context.Context, sessionID string, upda } // AddLibraryItemFileFromURI adds a file from a remote URI. -func (c *Manager) AddLibraryItemFileFromURI( - ctx context.Context, - sessionID, fileName, uri string) (*UpdateFile, error) { +func (c *Manager) AddLibraryItemFileFromURI(ctx context.Context, sessionID, name, uri string) (*UpdateFile, error) { + source := &TransferEndpoint{ + URI: uri, + } - n, fingerprint, err := c.getContentLengthAndFingerprint(ctx, uri) - if err != nil { - return nil, err + file := UpdateFile{ + Name: name, + SourceType: "PULL", + SourceEndpoint: source, } - info, err := c.AddLibraryItemFile(ctx, sessionID, UpdateFile{ - Name: fileName, - SourceType: "PULL", - Size: n, - SourceEndpoint: &TransferEndpoint{ - URI: uri, - SSLCertificateThumbprint: fingerprint, - }, - }) - if err != nil { - return nil, err + if res, err := c.Head(uri); err == nil { + file.Size = res.ContentLength + if res.TLS != nil { + source.SSLCertificateThumbprint = soap.ThumbprintSHA1(res.TLS.PeerCertificates[0]) + } + } else { + res, err := c.ProbeTransferEndpoint(ctx, *source) + if err != nil { + return nil, err + } + source.SSLCertificateThumbprint = res.SSLThumbprint } - return info, c.CompleteLibraryItemUpdateSession(ctx, sessionID) + return c.AddLibraryItemFile(ctx, sessionID, file) } // GetLibraryItemUpdateSessionFile retrieves information about a specific file @@ -102,25 +124,36 @@ func (c *Manager) GetLibraryItemUpdateSessionFile(ctx context.Context, sessionID return &res, c.Do(ctx, url.Request(http.MethodPost, spec), &res) } -// getContentLengthAndFingerprint gets the number of bytes returned -// by the URI as well as the SHA1 fingerprint of the peer certificate -// if the URI's scheme is https. -func (c *Manager) getContentLengthAndFingerprint( - ctx context.Context, uri string) (int64, string, error) { - resp, err := c.Head(uri) - if err != nil { - return 0, "", err - } - if resp.TLS == nil || len(resp.TLS.PeerCertificates) == 0 { - return resp.ContentLength, "", nil - } - fingerprint := c.Thumbprint(resp.Request.URL.Host) - if fingerprint == "" { - if c.DefaultTransport().TLSClientConfig.InsecureSkipVerify { - fingerprint = soap.ThumbprintSHA1(resp.TLS.PeerCertificates[0]) - } - } - return resp.ContentLength, fingerprint, nil +// ListLibraryItemUpdateSessionFile lists all files in the library item associated with the update session +func (c *Manager) ListLibraryItemUpdateSessionFile(ctx context.Context, sessionID string) ([]UpdateFile, error) { + url := c.Resource(internal.LibraryItemUpdateSessionFile).WithParam("update_session_id", sessionID) + var res []UpdateFile + return res, c.Do(ctx, url.Request(http.MethodGet), &res) +} + +// ValidateLibraryItemUpdateSessionFile validates all files in the library item associated with the update session +func (c *Manager) ValidateLibraryItemUpdateSessionFile(ctx context.Context, sessionID string) (*UpdateFileValidation, error) { + url := c.Resource(internal.LibraryItemUpdateSessionFile).WithID(sessionID).WithAction("validate") + var res UpdateFileValidation + return &res, c.Do(ctx, url.Request(http.MethodPost), &res) +} + +// RemoveLibraryItemUpdateSessionFile requests a file to be removed. The file will only be effectively removed when the update session is completed. +func (c *Manager) RemoveLibraryItemUpdateSessionFile(ctx context.Context, sessionID string, fileName string) error { + url := c.Resource(internal.LibraryItemUpdateSessionFile).WithID(sessionID).WithAction("remove") + spec := struct { + Name string `json:"file_name"` + }{fileName} + return c.Do(ctx, url.Request(http.MethodPost, spec), nil) +} + +func (c *Manager) ProbeTransferEndpoint(ctx context.Context, endpoint TransferEndpoint) (*ProbeResult, error) { + url := c.Resource(internal.LibraryItemUpdateSessionFile).WithAction("probe") + spec := struct { + SourceEndpoint TransferEndpoint `json:"source_endpoint"` + }{endpoint} + var res ProbeResult + return &res, c.Do(ctx, url.Request(http.MethodPost, spec), &res) } // ReadManifest converts an ovf manifest to a map of file name -> Checksum. diff --git a/vapi/simulator/simulator.go b/vapi/simulator/simulator.go old mode 100755 new mode 100644 index 43a6e4617..af1ef9df5 --- a/vapi/simulator/simulator.go +++ b/vapi/simulator/simulator.go @@ -55,6 +55,7 @@ import ( "github.com/vmware/govmomi/view" "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/methods" + "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" vim "github.com/vmware/govmomi/vim25/types" "github.com/vmware/govmomi/vim25/xml" @@ -1342,8 +1343,88 @@ func (s *handler) libraryItemUpdateSessionID(w http.ResponseWriter, r *http.Requ } } +func (s *handler) libraryItemProbe(endpoint library.TransferEndpoint) *library.ProbeResult { + p := &library.ProbeResult{ + Status: "SUCCESS", + } + + result := func() *library.ProbeResult { + for i, m := range p.ErrorMessages { + p.ErrorMessages[i].DefaultMessage = fmt.Sprintf(m.DefaultMessage, m.Args[0]) + } + return p + } + + u, err := url.Parse(endpoint.URI) + if err != nil { + p.Status = "INVALID_URL" + p.ErrorMessages = []rest.LocalizableMessage{{ + Args: []string{endpoint.URI}, + ID: "com.vmware.vdcs.cls-main.invalid_url_format", + DefaultMessage: "Invalid URL format for %s", + }} + return result() + } + + if u.Scheme != "http" && u.Scheme != "https" { + p.Status = "INVALID_URL" + p.ErrorMessages = []rest.LocalizableMessage{{ + Args: []string{endpoint.URI}, + ID: "com.vmware.vdcs.cls-main.file_probe_unsupported_uri_scheme", + DefaultMessage: "The specified URI %s is not supported", + }} + return result() + } + + res, err := http.Head(endpoint.URI) + if err != nil { + id := "com.vmware.vdcs.cls-main.http_request_error" + p.Status = "INVALID_URL" + + if soap.IsCertificateUntrusted(err) { + var info object.HostCertificateInfo + _ = info.FromURL(u, nil) + + id = "com.vmware.vdcs.cls-main.http_request_error_peer_not_authenticated" + p.Status = "CERTIFICATE_ERROR" + p.SSLThumbprint = info.ThumbprintSHA1 + } + + p.ErrorMessages = []rest.LocalizableMessage{{ + Args: []string{err.Error()}, + ID: id, + DefaultMessage: "HTTP request error: %s", + }} + + return result() + } + _ = res.Body.Close() + + if res.TLS != nil { + p.SSLThumbprint = soap.ThumbprintSHA1(res.TLS.PeerCertificates[0]) + } + + return result() +} + func (s *handler) libraryItemUpdateSessionFile(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { + switch r.Method { + case http.MethodPost: + switch s.action(r) { + case "probe": + var spec struct { + SourceEndpoint library.TransferEndpoint `json:"source_endpoint"` + } + if s.decode(r, w, &spec) { + res := s.libraryItemProbe(spec.SourceEndpoint) + OK(w, res) + } + default: + http.NotFound(w, r) + } + return + case http.MethodGet: + default: w.WriteHeader(http.StatusMethodNotAllowed) return } @@ -1443,10 +1524,20 @@ func (s *handler) libraryItemUpdateSessionFileID(w http.ResponseWriter, r *http. } OK(w, ids) case "remove": + if up.State != "ACTIVE" { + s.error(w, fmt.Errorf("removeFile not allowed in state %s", up.State)) + return + } delete(s.Update, id) OK(w) case "validate": - // TODO + if up.State != "ACTIVE" { + BadRequest(w, "com.vmware.vapi.std.errors.not_allowed_in_current_state") + return + } + var res library.UpdateFileValidation + // TODO check missing_files, validate .ovf + OK(w, res) } }