Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reposupport #69

Merged
merged 26 commits into from
Jul 7, 2022
Merged

Reposupport #69

Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
41f50e9
Add support for Artifact Download from local repo and other online re…
Jun 6, 2022
96a98af
changed variable name
Jun 6, 2022
686d3fa
Local repo and other online repo support for Download
Jun 6, 2022
b7ad166
changed varible names
Jun 8, 2022
6d6c8b1
l3afd/doc/api/README.md updated
Jun 9, 2022
113ef1e
comment changes
Jun 9, 2022
f4d2d81
some code cleaning and docs/api/README.md updated
Jun 15, 2022
7913b96
replaced multiple if-else with switch
Jun 15, 2022
f9ffeac
updated the doc
Jun 17, 2022
ce1d173
Update docs/api/README.md
Atul-source Jun 18, 2022
885ed7d
Update kf/bpf.go
Atul-source Jun 18, 2022
8dc8135
Update kf/bpf.go
Atul-source Jun 18, 2022
cc67c49
changes according to the reviews
Jun 18, 2022
31aa153
resolved security issue
Jun 18, 2022
2e567ef
validated function
Jun 20, 2022
fe8d15d
Merge branch 'main' into reposupport
Jun 20, 2022
012f877
fixed the doc
Jun 21, 2022
9cf4765
Merge branch 'reposupport' of https://github.com/Atul-source/l3afd in…
Jun 21, 2022
d495bf2
added tests
Jun 23, 2022
370f5a7
tests
Jun 23, 2022
c4b79ac
some typo correction
Jun 23, 2022
8985195
added some mocks
Jun 28, 2022
c00e56d
resolving merge conflict
Jun 28, 2022
52223d8
resolving conflict
Jun 28, 2022
8bed18f
removed unused file
Jun 28, 2022
cf51871
Update kf/bpf_test.go
dthaler Jul 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/api/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ The payload will look more like this standard JSON:
"name": "ratelimiting",
"seq_id": 1,
"artifact": "l3af_ratelimiting.tar.gz",
"ebpf_package_repo_url": "https://l3af.io"
"map_name": "/sys/fs/bpf/xdp_rl_ingress_next_prog",
"cmd_start": "ratelimiting",
"version": "latest",
Expand Down Expand Up @@ -46,7 +47,8 @@ The payload will look more like this standard JSON:
|---------------------|------------------------------------------------|----------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------|
| name | string | ratelimiting | Name of the eBPF Program |
| seq_id | number | `1` | Position of the eBPF program in the chain. Count starts at 1. |
| artifact | string | `"l3af_ratelimiting.tar.gz"` | Userspace eBPF program binary and kernel eBPF byte code in tar.gz format |
| artifact | string | `"l3af_ratelimiting.tar.gz"` | Userspace eBPF program binary and kernel eBPF byte code in tar.gz format |
| ebpf_package_repo_url | string | `"https://l3af.io/"` | eBPF package repository URL. If it is not provided default URL is used.| |
| map_name | string | `"/sys/fs/bpf/ep1_next_prog_array"` | Chaining program map in the file system with path. This should match the eBPF program code. |
| cmd_start | string | `"ratelimiting"` | The command used to start the eBPF program. Usually the userspace eBPF program binary name. |
| cmd_stop | string | | The command used stop the eBPF program |
Expand Down
247 changes: 147 additions & 100 deletions kf/bpf.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ var (
//lint:ignore U1000 avoid false linter error on windows, since this variable is only used in linux code
const executePerm uint32 = 0111
const bpfStatus string = "RUNNING"
const httpScheme string = "http"
const httpsScheme string = "https"
const fileScheme string = "file"

// BPF defines run time details for BPFProgram.
type BPF struct {
Expand Down Expand Up @@ -520,139 +523,172 @@ func (b *BPF) VerifyAndGetArtifacts(conf *config.Config) error {
return nil
}

// GetArtifacts downloads artifacts from the nexus repo
// GetArtifacts downloads artifacts from the specified eBPF repo
func (b *BPF) GetArtifacts(conf *config.Config) error {
var fPath = ""

kfRepoURL, err := url.Parse(conf.KFRepoURL)
if err != nil {
return fmt.Errorf("unknown KF repo url format: %w", err)
buf := &bytes.Buffer{}
isDefaultURLUsed := false
RepoURL := b.Program.EPRURL
if len(b.Program.EPRURL) == 0 {
RepoURL = conf.KFRepoURL
isDefaultURLUsed = true
}

platform, err := GetPlatform()
URL, err := url.Parse(RepoURL)
if err != nil {
return fmt.Errorf("failed to find KF repo download path: %w", err)
if isDefaultURLUsed {
return fmt.Errorf("unknown kf-repo format : %w", err)
} else {
return fmt.Errorf("unknown ebpf_package_repo_url format : %w", err)
}
}

kfRepoURL.Path = path.Join(kfRepoURL.Path, b.Program.Name, b.Program.Version, platform, b.Program.Artifact)
log.Info().Msgf("Downloading - %s", kfRepoURL)
switch URL.Scheme {
case httpsScheme, httpScheme:
{
platform, err := GetPlatform()
if err != nil {
return fmt.Errorf("failed to identify platform type: %w", err)
}

timeOut := time.Duration(conf.HttpClientTimeout) * time.Second
var netTransport = &http.Transport{
ResponseHeaderTimeout: timeOut,
}
client := http.Client{Transport: netTransport, Timeout: timeOut}
URL.Path = path.Join(URL.Path, b.Program.Name, b.Program.Version, platform, b.Program.Artifact)
log.Info().Msgf("Downloading - %s", URL)

// Get the data
resp, err := client.Get(kfRepoURL.String())
if err != nil {
return fmt.Errorf("download failed: %w", err)
}
defer resp.Body.Close()
timeOut := time.Duration(conf.HttpClientTimeout) * time.Second
var netTransport = &http.Transport{
ResponseHeaderTimeout: timeOut,
}
client := http.Client{Transport: netTransport, Timeout: timeOut}

buf := &bytes.Buffer{}
buf.ReadFrom(resp.Body)
// Get the data
resp, err := client.Get(URL.String())
if err != nil {
return fmt.Errorf("download failed: %w", err)
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return fmt.Errorf("get request returned unexpected status code: %d (%s), %d was expected\n\tResponse Body: %s", resp.StatusCode, http.StatusText(resp.StatusCode), http.StatusOK, buf.Bytes())
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("get request returned unexpected status code: %d (%s), %d was expected\n\tResponse Body: %s", resp.StatusCode, http.StatusText(resp.StatusCode), http.StatusOK, buf.Bytes())
}

buf.ReadFrom(resp.Body)

if strings.HasSuffix(b.Program.Artifact, ".zip") {
c := bytes.NewReader(buf.Bytes())
zipReader, err := zip.NewReader(c, int64(c.Len()))
if err != nil {
return fmt.Errorf("failed to create zip reader: %w", err)
}
tempDir := filepath.Join(conf.BPFDir, b.Program.Name, b.Program.Version)
case fileScheme:
{
if fileExists(URL.Path) {
f, err := os.Open(URL.Path)
if err != nil {
return fmt.Errorf("opening err : %w", err)
}

for _, file := range zipReader.File {
buf.ReadFrom(f)
f.Close()
} else {
return fmt.Errorf("artifact is not found")
}
}
}

zippedFile, err := file.Open()
switch artifact := b.Program.Artifact; {
case strings.HasSuffix(artifact, ".zip"):
{
c := bytes.NewReader(buf.Bytes())
zipReader, err := zip.NewReader(c, int64(c.Len()))
if err != nil {
return fmt.Errorf("unzip failed: %w", err)
return fmt.Errorf("failed to create zip reader: %w", err)
}
defer zippedFile.Close()

extractedFilePath := filepath.Join(
tempDir,
file.Name,
)
if !strings.HasPrefix(extractedFilePath, filepath.Clean(tempDir)+string(os.PathSeparator)) {
return fmt.Errorf("invalid file path: %s", extractedFilePath)
}
if file.FileInfo().IsDir() {
os.MkdirAll(extractedFilePath, file.Mode())
} else {
outputFile, err := os.OpenFile(
extractedFilePath,
os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
file.Mode(),
)
tempDir := filepath.Join(conf.BPFDir, b.Program.Name, b.Program.Version)

for _, file := range zipReader.File {

zippedFile, err := file.Open()
if err != nil {
return fmt.Errorf("unzip failed to create file: %w", err)
return fmt.Errorf("unzip failed: %w", err)
}
defer outputFile.Close()
defer zippedFile.Close()

buf := copyBufPool.Get().(*bytes.Buffer)
_, err = io.CopyBuffer(outputFile, zippedFile, buf.Bytes())
extractedFilePath, err := ValidatePath(file.Name, tempDir)
if err != nil {
return fmt.Errorf("GetArtifacts failed to copy files: %w", err)
return err
}

if file.FileInfo().IsDir() {
os.MkdirAll(extractedFilePath, file.Mode())
} else {
outputFile, err := os.OpenFile(
extractedFilePath,
os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
file.Mode(),
)
if err != nil {
return fmt.Errorf("unzip failed to create file: %w", err)
}
defer outputFile.Close()

buf := copyBufPool.Get().(*bytes.Buffer)
_, err = io.CopyBuffer(outputFile, zippedFile, buf.Bytes())
if err != nil {
return fmt.Errorf("GetArtifacts failed to copy files: %w", err)
}
copyBufPool.Put(buf)
}
copyBufPool.Put(buf)
}
newDir := strings.Split(b.Program.Artifact, ".")
b.FilePath = filepath.Join(tempDir, newDir[0])
return nil
}
newDir := strings.Split(b.Program.Artifact, ".")
b.FilePath = filepath.Join(tempDir, newDir[0])
return nil
} else if strings.HasSuffix(b.Program.Artifact, ".tar.gz") {
archive, err := gzip.NewReader(buf)
if err != nil {
return fmt.Errorf("failed to create Gzip reader: %w", err)
}
defer archive.Close()
tarReader := tar.NewReader(archive)
tempDir := filepath.Join(conf.BPFDir, b.Program.Name, b.Program.Version)
case strings.HasSuffix(b.Program.Artifact, ".tar.gz"):
{
archive, err := gzip.NewReader(buf)
if err != nil {
return fmt.Errorf("failed to create Gzip reader: %w", err)
}
defer archive.Close()
tarReader := tar.NewReader(archive)
tempDir := filepath.Join(conf.BPFDir, b.Program.Name, b.Program.Version)

for {
header, err := tarReader.Next()
for {
header, err := tarReader.Next()

if err == io.EOF {
break
} else if err != nil {
return fmt.Errorf("untar failed: %w", err)
}
if err == io.EOF {
break
} else if err != nil {
return fmt.Errorf("untar failed: %w", err)
}

if strings.Contains(header.Name, "..") {
return fmt.Errorf("zipped file contians filepath (%s) that includes (..)", header.Name)
}
fPath, err := ValidatePath(header.Name, tempDir)
if err != nil {
return err
}

fPath = filepath.Join(tempDir, header.Name)
info := header.FileInfo()
if info.IsDir() {
if err = os.MkdirAll(fPath, info.Mode()); err != nil {
return fmt.Errorf("untar failed to create directories: %w", err)
info := header.FileInfo()
if info.IsDir() {
if err = os.MkdirAll(fPath, info.Mode()); err != nil {
return fmt.Errorf("untar failed to create directories: %w", err)
}
continue
}
continue
}

file, err := os.OpenFile(fPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
if err != nil {
return fmt.Errorf("untar failed to create file: %w", err)
}
defer file.Close()
file, err := os.OpenFile(fPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
if err != nil {
return fmt.Errorf("untar failed to create file: %w", err)
}
defer file.Close()

buf := copyBufPool.Get().(*bytes.Buffer)
_, err = io.CopyBuffer(file, tarReader, buf.Bytes())
if err != nil {
return fmt.Errorf("GetArtifacts failed to copy files: %w", err)
buf := copyBufPool.Get().(*bytes.Buffer)
_, err = io.CopyBuffer(file, tarReader, buf.Bytes())
if err != nil {
return fmt.Errorf("GetArtifacts failed to copy files: %w", err)
}
copyBufPool.Put(buf)
}
copyBufPool.Put(buf)
newDir := strings.Split(b.Program.Artifact, ".")
b.FilePath = filepath.Join(tempDir, newDir[0])
return nil
}
newDir := strings.Split(b.Program.Artifact, ".")
b.FilePath = filepath.Join(tempDir, newDir[0])
return nil
} else {
return fmt.Errorf("unknown artifact format ")
default:
return fmt.Errorf("unknown artifact format")
}
}

Expand Down Expand Up @@ -1001,3 +1037,14 @@ func (b *BPF) VerifyMetricsMapsVanish() error {
log.Error().Err(err).Msg("")
return err
}

func ValidatePath(filePath string, destination string) (string, error) {
destpath := filepath.Join(destination, filePath)
if strings.Contains(filePath, "..") {
return "", fmt.Errorf(" file contains filepath (%s) that includes (..)", filePath)
}
if !strings.HasPrefix(destpath, filepath.Clean(destination)+string(os.PathSeparator)) {
return "", fmt.Errorf("%s: illegal file path", filePath)
}
return destpath, nil
}
53 changes: 27 additions & 26 deletions models/l3afd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,32 +23,33 @@ type L3afDNFArgs map[string]interface{}

// BPFProgram defines BPF Program for specific host
type BPFProgram struct {
ID int `json:"id"` // Program id
Name string `json:"name"` // Name of the BPF program
SeqID int `json:"seq_id"` // Sequence position in the chain
Artifact string `json:"artifact"` // Artifact file name
MapName string `json:"map_name"` // BPF map to store next program fd
CmdStart string `json:"cmd_start"` // Program start command
CmdStop string `json:"cmd_stop"` // Program stop command
CmdStatus string `json:"cmd_status"` // Program status command
CmdConfig string `json:"cmd_config"` // Program config providing command
Version string `json:"version"` // Program version
UserProgramDaemon bool `json:"user_program_daemon"` // User program daemon or not
IsPlugin bool `json:"is_plugin"` // User program is plugin or not
CPU int `json:"cpu"` // User program cpu limits
Memory int `json:"memory"` // User program memory limits
AdminStatus string `json:"admin_status"` // Program admin status enabled or disabled
ProgType string `json:"prog_type"` // Program type XDP or TC
RulesFile string `json:"rules_file"` // Config rules file name
Rules string `json:"rules"` // Config rules
ConfigFilePath string `json:"config_file_path"` // Config file location
CfgVersion int `json:"cfg_version"` // Config version
StartArgs L3afDNFArgs `json:"start_args"` // Map of arguments to start command
StopArgs L3afDNFArgs `json:"stop_args"` // Map of arguments to stop command
StatusArgs L3afDNFArgs `json:"status_args"` // Map of arguments to status command
MapArgs L3afDNFArgs `json:"map_args"` // Config BPF Map of arguments
ConfigArgs L3afDNFArgs `json:"config_args"` // Map of arguments to config command
MonitorMaps []L3afDNFMetricsMap `json:"monitor_maps"` // Metrics BPF maps
ID int `json:"id"` // Program id
Name string `json:"name"` // Name of the BPF program
SeqID int `json:"seq_id"` // Sequence position in the chain
Artifact string `json:"artifact"` // Artifact file name
MapName string `json:"map_name"` // BPF map to store next program fd
CmdStart string `json:"cmd_start"` // Program start command
CmdStop string `json:"cmd_stop"` // Program stop command
CmdStatus string `json:"cmd_status"` // Program status command
CmdConfig string `json:"cmd_config"` // Program config providing command
Version string `json:"version"` // Program version
UserProgramDaemon bool `json:"user_program_daemon"` // User program daemon or not
IsPlugin bool `json:"is_plugin"` // User program is plugin or not
CPU int `json:"cpu"` // User program cpu limits
Memory int `json:"memory"` // User program memory limits
AdminStatus string `json:"admin_status"` // Program admin status enabled or disabled
ProgType string `json:"prog_type"` // Program type XDP or TC
RulesFile string `json:"rules_file"` // Config rules file name
Rules string `json:"rules"` // Config rules
ConfigFilePath string `json:"config_file_path"` // Config file location
CfgVersion int `json:"cfg_version"` // Config version
StartArgs L3afDNFArgs `json:"start_args"` // Map of arguments to start command
StopArgs L3afDNFArgs `json:"stop_args"` // Map of arguments to stop command
StatusArgs L3afDNFArgs `json:"status_args"` // Map of arguments to status command
MapArgs L3afDNFArgs `json:"map_args"` // Config BPF Map of arguments
ConfigArgs L3afDNFArgs `json:"config_args"` // Map of arguments to config command
MonitorMaps []L3afDNFMetricsMap `json:"monitor_maps"` // Metrics BPF maps
EPRURL string `json:"ebpf_package_repo_url"` // Download url for Program
}

// L3afDNFMetricsMap defines BPF map
Expand Down