Merge Dev

.github/workflows/test-mlc-script-features.yml

@@ -113,7 +113,9 @@ jobs:
 
         mlcr --tags=install,python-venv --version=3.10.8 --name=mlperf --quiet
 
-        mlcr --tags=run,mlperf,inference,_submission,_short --adr.python.name=mlperf --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --submitter=MLCommons --implementation=cpp --hw_name=default --model=retinanet --backend=onnxruntime --device=cpu --scenario=Offline --quiet
+        export MLC_SCRIPT_EXTRA_CMD="--adr.python.name=mlperf"
+
+        mlcr --tags=run,mlperf,inference,_submission,_short --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --submitter=MLCommons --implementation=cpp --hw_name=default --model=retinanet --backend=onnxruntime --device=cpu --scenario=Offline --quiet
 
      # Step for Linux/MacOS
     - name: Randomly Execute Step (Linux/MacOS)

.github/workflows/test-mlperf-inference-abtf-poc.yml

@@ -10,6 +10,7 @@ on:
 
 jobs:
   build:
+    name: Test MLPerf Inference ABTF POC
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml

@@ -10,6 +10,7 @@ on:
 
 jobs:
   build:
+    name: MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml

@@ -10,6 +10,7 @@ on:
 
 jobs:
   build:
+    name: MLPerf inference MLCommons C++ ResNet50
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

.github/workflows/test-mlperf-inference-rgat.yml

@@ -9,7 +9,8 @@ on:
       - '!**.md'
 
 jobs:
-  build:
+  rgat-inference-run:
+    name:  ${{ matrix.os }} - ${{ matrix.backend }} - ${{ matrix.implementation }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

.github/workflows/test-mlperf-inference-tvm-resnet50.yml

@@ -10,6 +10,7 @@ on:
 
 jobs:
   tvm-run:
+    name: TVM ResNet50
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false

.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   build:
-
+    name: Test MLPerf loadgen with HuggingFace bert onnx fp32 squad model on Python ${{ matrix.python-version }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false

.github/workflows/test-nvidia-mlperf-inference-implementations.yml

@@ -17,12 +17,14 @@ jobs:
       strategy:
         fail-fast: false
         matrix:
-          system: [ "GO-spr", "phoenix"]
+          system: [ "GO-spr", "phoenix", "GO-i9" ]
           # system: [ "mlc-server" ] 
           python-version: [ "3.12" ]
           model: [ "resnet50",  "retinanet",  "bert-99", "bert-99.9", "gptj-99.9", "3d-unet-99.9", "sdxl" ]
           exclude:
            - model: gptj-99.9
+           - system: phoenix
+           - system: GO-i9
 
       steps:
       - name: Test MLPerf Inference NVIDIA ${{ matrix.model }}
@@ -43,12 +45,7 @@ jobs:
             gpu_name=rtx_4090
             docker_string=" --docker"
           fi
-          if [ "${{ matrix.model }}" = "bert-99" ] || [ "${{ matrix.model }}" = "bert-99.9" ]; then
-            category="edge"
-          else
-            category="datacenter,edge"
-          fi
-
+          category="datacenter,edge"
           if [ -f "gh_action/bin/deactivate" ]; then source gh_action/bin/deactivate; fi
           python3 -m venv gh_action
           source gh_action/bin/activate

automation/script/docker_utils.py

@@ -109,7 +109,7 @@ def prepare_docker_inputs(input_params, docker_settings,
         keys += [
             "skip_run_cmd", "pre_run_cmds", "run_cmd_prefix", "all_gpus", "num_gpus", "device", "gh_token",
             "port_maps", "shm_size", "pass_user_id", "pass_user_group", "extra_run_args", "detached", "interactive",
-            "dt", "it", "use_host_group_id", "use_host_user_id", "keep_detached", "reuse_existing"
+            "dt", "it", "use_host_group_id", "use_host_user_id", "keep_detached", "reuse_existing", "use_google_dns"
         ]
     # Collect Dockerfile inputs
     docker_inputs = {
@@ -413,7 +413,7 @@ def get_container_path_script(i):
 def get_container_path(value, username="mlcuser"):
     path_split = value.split(os.sep)
     if len(path_split) == 1:
-        return value
+        return value, value
 
     new_value = ''
     if "cache" in path_split and "local" in path_split:

automation/script/module.py

@@ -273,31 +273,17 @@ def _run(self, i):
         # Useful to set up default flags such as the name of virtual enviroment
         extra_cli = os.environ.get('MLC_SCRIPT_EXTRA_CMD', '').strip()
         if extra_cli != '':
-            from cmind import cli
-            r = cli.parse(extra_cli)
+            r = convert_args_to_dictionary(extra_cli)
             if r['return'] > 0:
                 return r
 
-            mlc_input = r['mlc_input']
+            mlc_input = r['args_dict']
 
             utils.merge_dicts({'dict1': i,
                                'dict2': mlc_input,
                                'append_lists': True,
                                'append_unique': True})
 
-        # Check if has extra tags as a second artifact
-        # Example: cmr . "_python _tiny"
-
-        parsed_artifacts = i.get('parsed_artifacts', [])
-        if len(parsed_artifacts) > 0:
-            extra_tags = parsed_artifacts[0][0][0]
-            if ' ' in extra_tags or ',' in extra_tags:
-                # Add tags
-                x = i.get('tags', '')
-                if x != '':
-                    x += ','
-                i['tags'] = x + extra_tags.replace(' ', ',')
-
         # Recursion spaces needed to format log and print
         recursion_spaces = i.get('recursion_spaces', '')
         # Caching selections to avoid asking users again

automation/utils.py

@@ -259,7 +259,6 @@ def unzip_file(i):
 
     console = i.get('out') == 'con'
 
-    # Attempt to read cmr.json
     file_name_handle = open(file_name, 'rb')
     file_name_zip = zipfile.ZipFile(file_name_handle)
 

flow.md

@@ -3,7 +3,11 @@
 ```mermaid
 flowchart TD
     A[MLC interface] --> B{Script Automation}
-    A[MLC interface] --> C{Cache Automation}
-    B[Script Automation] --> C{Cache Automation}
+    A[MLC interface] --> C{Cache Action}
+    B[Script Automation] --> C{Cache Action}
+    A[MLC interface] --> E{Docker Action}
+    B[Script Automation] --> E{Docker Action}
+    A[MLC interface] --> F{Test Action}
+    B[Script Automation] --> F{Test Action}
     B[Script Automation] --> D{MLPerf Scripts}
 ```

script/app-mlperf-inference/customize.py

@@ -239,7 +239,7 @@ def postprocess(i):
         sut_config[model_full_name][scenario][metric] = value
 
         print(
-            f"SUT: {sut_name}, model: {model_full_name}, scenario: {scenario}, {metric} updated as {value}")
+            f"SUT: {sut_name}, model: {model_full_name}, scenario: {scenario}, {metric} (mean value) updated as {value}")
         print(f"New config stored in {sut_config_path}")
         with open(sut_config_path, "w") as f:
             yaml.dump(sut_config, f)

script/app-mlperf-inference/meta.yaml

@@ -113,7 +113,7 @@ deps:
       - 'yes'
   - tags: get,mlperf,inference,utils
 
-  - tags: install,pip-package,for-cmind-python,_package.pandas
+  - tags: install,pip-package,for-mlc-python,_package.pandas
     enable_if_env:
       MLC_PROFILE_NVIDIA_POWER:
         - on
@@ -676,6 +676,8 @@ variations:
       model
     default:
       true
+    default_env:
+      MLC_USE_DATASET_FROM_HOST: yes
     env:
       MLC_MODEL:
         resnet50
@@ -1959,7 +1961,8 @@ docker:
   skip_run_cmd: 'no'
   shm_size: '32gb'
   interactive: True
-  extra_run_args: ' --dns 8.8.8.8 --dns 8.8.4.4 --cap-add SYS_ADMIN  --cap-add SYS_TIME --security-opt apparmor=unconfined --security-opt seccomp=unconfined'
+  extra_run_args: ' --cap-add SYS_ADMIN  --cap-add SYS_TIME --security-opt apparmor=unconfined --security-opt seccomp=unconfined'
+  use_google_dns: yes
   os: ubuntu
   user: mlcuser
   mlc_repo: mlcommons@mlperf-automations

script/clean-nvidia-mlperf-inference-scratch-space/customize.py

@@ -16,8 +16,9 @@ def preprocess(i):
 
     clean_cmd = ''
     cache_rm_tags = ''
-    extra_cache_rm_tags = env.get('MLC_CLEAN_EXTRA_CACHE_RM_TAGS', '')
+    extra_cache_rm_tags = env.get('MLC_CLEAN_EXTRA_CACHE_RM_TAGS', '').strip()
 
+    extra_tags = "," + extra_cache_rm_tags if extra_cache_rm_tags != '' else ''
     if env.get('MLC_MODEL', '') == 'sdxl':
         if env.get('MLC_CLEAN_ARTIFACT_NAME', '') == 'downloaded_data':
             clean_cmd = f"""rm -rf {os.path.join(env['MLC_NVIDIA_MLPERF_SCRATCH_PATH'], "data", "coco", "SDXL")} """
@@ -29,7 +30,7 @@ def preprocess(i):
             clean_cmd = f"""rm -rf {os.path.join(env['MLC_NVIDIA_MLPERF_SCRATCH_PATH'], "models", "SDXL")} """
             cache_rm_tags = "nvidia-harness,_download_model,_sdxl"
 
-    cache_rm_tags = cache_rm_tags + extra_cache_rm_tags
+    cache_rm_tags = cache_rm_tags + extra_tags
     mlc_cache = i['automation'].cache_action
 
     if cache_rm_tags:

script/generate-mlperf-inference-submission/meta.yaml

@@ -17,7 +17,7 @@ deps:
   - inference-src
   tags: mlcommons,inference,src
 - tags: get,sut,system-description
-- tags: install,pip-package,for-cmind-python,_package.tabulate
+- tags: install,pip-package,for-mlc-python,_package.tabulate
 - tags: get,mlperf,inference,utils
 - names:
   - get-mlperf-results-dir

script/get-dataset-igbh/customize.py

@@ -43,7 +43,7 @@ def preprocess(i):
     run_cmd += x_sep + \
         f"""{
             env['MLC_PYTHON_BIN_WITH_PATH']} tools/split_seeds.py --path {download_loc} --dataset_size {
-            env['MLC_DATASET_IGBH_SIZE']} """
+            env['MLC_DATASET_IGBH_SIZE']} {env.get('MLC_IGBH_CALIBRATION_FLAG', '')} """
 
     # compress graph(for glt implementation)
     if env.get('MLC_IGBH_GRAPH_COMPRESS', '') == "yes":

script/get-dataset-igbh/meta.yaml

@@ -437,6 +437,9 @@ variations:
   glt:
     env:
       MLC_IGBH_GRAPH_COMPRESS: yes
+  calibration:
+    env:
+      MLC_IGBH_CALIBRATION_FLAG: --calibration
   csc:
     group: compressed-layout
     default: true

script/get-generic-sys-util/meta.yaml

@@ -690,6 +690,20 @@ variations:
         brew: transmission
         dnf: transmission-daemon
         yum: transmission-daemon
+  unzip:
+    env:
+      MLC_SYS_UTIL_NAME: unzip
+      MLC_SYS_UTIL_VERSION_CMD: unzip --version
+    new_env_keys:
+    - MLC_UNZIP_VERSION
+    state:
+      unzip:
+        apt: unzip
+        brew: unzip
+        dnf: unzip
+        yum: unzip
+        choco: unzip
+
   vim-common:
     env:
       MLC_SYS_UTIL_NAME: vim_common

script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml

@@ -39,4 +39,4 @@
     Offline:
       target_qps: 1.3
     Server:
-      target_qps: 0.6
+      target_qps: 0.4

script/get-platform-details/meta.yaml

@@ -54,6 +54,7 @@ prehook_deps:
     MLC_SUDO_USER:
     - 'no'
   tags: get,sys-util,generic,_linux-tools
+  ignore_missing: True
 tags:
 - get
 - platform

script/get-sys-utils-min/customize.py

@@ -1,5 +1,6 @@
 from mlc import utils
 import os
+from utils import download_file, unzip_file
 
 
 def preprocess(i):
@@ -22,7 +23,7 @@ def preprocess(i):
             for cd in clean_dirs.split(','):
                 if cd != '':
                     if os.path.isdir(cd):
-                        print('Clearning directory {}'.format(cd))
+                        print('Cleaning directory {}'.format(cd))
                         shutil.rmtree(cd)
 
         url = env['MLC_PACKAGE_WIN_URL']
@@ -38,16 +39,25 @@ def preprocess(i):
 
             print('')
             print('Downloading from {}'.format(url))
-            env['MLC_DAE_FINAL_ENV_NAME'] = 'FILENAME'
-            env['MLC_OUTDIRNAME'] = os.getcwd()
-            r = cm.access({'action': 'run',
-                           'target': 'script',
-                           'env': env,
-                           'tags': 'download-and-extract,_extract',
-                           'url': url})
+            r = download_file({
+                'url': url,
+                'verify': False})
             if r['return'] > 0:
                 return r
 
+            filename = r['filename']
+
+            print('Unzipping file {}'.format(filename))
+
+            r = unzip_file({
+                'filename': filename})
+            if r['return'] > 0:
+                return r
+
+            if os.path.isfile(filename):
+                print('Removing file {}'.format(filename))
+                os.remove(filename)
+
         print('')
 
         # Add to path

script/install-cmake-prebuilt/customize.py

@@ -71,8 +71,6 @@ def preprocess(i):
     print('')
     print('Downloading from {} ...'.format(package_url))
 
-    cm = automation.action_object
-
     r = download_file({
         'url': package_url})
     if r['return'] > 0:

script/install-pip-package-for-cmind-python/meta.yaml → script/install-pip-package-for-mlc-python/meta.yaml

@@ -1,4 +1,4 @@
-alias: install-pip-package-for-cmind-python
+alias: install-pip-package-for-mlc-python
 automation_alias: script
 automation_uid: 5b4e0237da074764
 cache: true
@@ -7,8 +7,8 @@ tags:
 - pip
 - package
 - pip-package
-- for-cmind-python
-- for.cmind-python
+- for-mlc-python
+- for.mlc-python
 uid: b16ed087abab459c
 
 variations:

script/run-docker-container/customize.py

@@ -56,7 +56,7 @@ def preprocess(i):
     print('')
     # CMD = f"""{env['MLC_CONTAINER_TOOL']} ps --format=json  --filter "ancestor={DOCKER_CONTAINER}" """
     CMD = f"""{env['MLC_CONTAINER_TOOL']} ps --format """ + \
-        "'{{ .ID }},'" + f"""  --filter "ancestor={DOCKER_CONTAINER}" """
+        '"{{ .ID }},"' + f"""  --filter "ancestor={DOCKER_CONTAINER}" """
     if os_info['platform'] == 'windows':
         CMD += " 2> nul"
     else:
@@ -160,7 +160,8 @@ def postprocess(i):
         for mounts in env['MLC_DOCKER_VOLUME_MOUNTS']:
             mount_cmds.append(mounts)
 
-    if env.get('MLC_DOCKER_PASS_USER_GROUP', '') != '':
+    if env.get('MLC_DOCKER_PASS_USER_GROUP',
+               '') != '' and os_info['platform'] != 'windows':
         run_opts += " --group-add $(id -g $USER) "
 
     if env.get('MLC_DOCKER_ADD_DEVICE', '') != '':
@@ -180,6 +181,9 @@ def postprocess(i):
     if env.get('MLC_DOCKER_EXTRA_RUN_ARGS', '') != '':
         run_opts += env['MLC_DOCKER_EXTRA_RUN_ARGS']
 
+    if is_true(env.get('MLC_DOCKER_USE_GOOGLE_DNS', '')):
+        run_opts += ' --dns 8.8.8.8 --dns 8.8.4.4 '
+
     if env.get('MLC_CONTAINER_TOOL', '') == 'podman' and env.get(
             'MLC_PODMAN_MAP_USER_ID', '').lower() not in ["no", "0", "false"]:
         run_opts += " --userns=keep-id"