Update requirements.txt.

broker/bash_scripts/clean_for_new_test.sh

@@ -65,7 +65,7 @@ rm -f $BASE/package-lock.json
 
 rm -rf docs/_build_html/
 rm -rf docs/_build/
-rm /tmp/run/driver_popen.pid
+rm -f /tmp/run/driver_popen.pid >/dev/null 2>&1
 rm -f ~/.ebloc-broker/.oc_client.pckl
 rm -f /var/ebloc-broker/cache/*.tar.gz
 # rm -f .oc.pckl

broker/bash_scripts/folder_setup.sh

@@ -89,6 +89,8 @@ if [ ! -d $LOG_DIR ]; then
     mkdir -p $LOG_DIR
 fi
 
+touch $LOG_DIR/.eudat_client.txt
+
 mkdir -p $LOG_DIR/private
 mkdir -p $LOG_DIR/drivers_output
 mkdir -p $LOG_DIR/links

broker/eblocbroker_scripts/job.py

@@ -430,17 +430,17 @@ def set_storage_cost(self, is_verbose=False):
                     if self.job.data_prices_set_block_numbers[idx] == 0:
                         registered_data_bn_list = self.Ebb.get_registered_data_bn(self.job.provider, code_hash)
                         if bn > registered_data_bn_list[-1]:
-                            data_price_set_block_number = registered_data_bn_list[-1]
+                            data_price_set_bn = registered_data_bn_list[-1]
                         else:
-                            data_price_set_block_number = registered_data_bn_list[-2]
+                            data_price_set_bn = registered_data_bn_list[-2]
                     else:
-                        data_price_set_block_number = self.job.data_prices_set_block_numbers[idx]
+                        data_price_set_bn = self.job.data_prices_set_block_numbers[idx]
 
                     # if true, registered data's price should be considered for storage
                     output = self.ebb.getRegisteredDataPrice(
                         self.job.provider,
                         code_hash,
-                        data_price_set_block_number,
+                        data_price_set_bn,
                     )
                     data_price = output[0]
                     self.storage_cost += data_price

broker/eblocbroker_scripts/process_payment.py

@@ -26,9 +26,14 @@ def process_payment(
     received_block_number=0,
 ):
     """Process payment of the received job."""
+    if not result_ipfs_hash:
+        _result_ipfs_hash = '""'
+    else:
+        _result_ipfs_hash = result_ipfs_hash
+
     log(
         f"~/ebloc-broker/broker/eblocbroker_scripts/process_payment.py {job_key} {index} {job_id} {elapsed_time}"
-        f" {result_ipfs_hash} '{cloud_storage_ids}' {ended_timestamp} {data_transfer_in} {data_transfer_out} '{core}'"
+        f" {_result_ipfs_hash} '{cloud_storage_ids}' {ended_timestamp} {data_transfer_in} {data_transfer_out} '{core}'"
         f" '{run_time}'",
         "bold blue",
     )
@@ -74,8 +79,7 @@ def process_payment(
     return self.tx_id(tx)
 
 
-if __name__ == "__main__":
-    Ebb = cfg.Ebb
+def main():
     if len(sys.argv) == 12:
         args = sys.argv[1:]
         my_args = []  # type: Union[Any]
@@ -106,7 +110,7 @@ def process_payment(
         sys.exit(1)
 
     try:
-        tx_hash = Ebb.process_payment(
+        tx_hash = cfg.Ebb.process_payment(
             job_key,
             index,
             job_id,
@@ -122,3 +126,7 @@ def process_payment(
         log(f"tx_hash={tx_hash}")
     except:
         sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

broker/eblocbroker_scripts/register_data.py

@@ -26,6 +26,11 @@ def _register_data(source_code_hash, data_price, commitment_dur):
     source_code_hash_bytes = cfg.w3.toBytes(text=source_code_hash)
     with suppress(Exception):
         (price, _commitment_dur) = cfg.Ebb.get_registered_data_price(env.PROVIDER_ID, source_code_hash_bytes, 0)
+        bn = cfg.Ebb.get_registered_data_bn(env.PROVIDER_ID, source_code_hash_bytes)
+        if bn[0] == 0:
+            log(f"E: registered block number returns zero for {source_code_hash_bytes}")
+            is_exit = True
+
         log(
             f"## data([green]{source_code_hash}[/green]) is already registerered.\n"
             "Use [blue]./update_data_price.py[/blue] to update its price"

broker/link.py

@@ -30,7 +30,7 @@ def umount(self, data_hashes):
                     run(["sudo", "umount", "-f", dest], is_quiet=True)
 
     def link(self, path, dest, is_read_only=False):
-        """Create links between folders.
+        """Create link between folders.
 
         You can create a read-only bind-mount(https://lwn.net/Articles/281157/).
         mount --bind /path/to/source/ /path/to/dest/

broker/test_setup/README.org

@@ -7,3 +7,51 @@
 :IMPORTANT: DO NOT download from ~Storage~ section, it download ~meta_data.json~ separately.
 
 - *For requesters:* Download from ~Storage~ zipped patches.
+
+----------------------------------------------------------------------------
+
+* NAS Parallel Benchmarks
+
+site: [[https://www.nas.nasa.gov/software/npb.html]]
+
+#+BEGIN_SRC bash
+wget https://www.nas.nasa.gov/assets/npb/NPB3.3.1.tar.gz
+tar -xvf NPB3.3.1.tar.gz
+cd NPB3.3.1/NPB3.3-SER
+cp config/make.def.template config/make.def
+
+make bt CLASS=B
+/usr/bin/time -v bin/bt.B.x inputbt.data
+
+make sp CLASS=B
+/usr/bin/time -v bin/sp.B.x inputsp.data
+
+make ua CLASS=B
+/usr/bin/time -v bin/ua.B.x inputua.data
+
+make lu CLASS=B
+/usr/bin/time -v bin/lu.B.x inputlu.data
+#+END_SRC
+
+~(/usr/bin/time -v bin/lu.B.x inputlu.data) >> output_.txt 2>&1~
+
+~CLASSES: [A and B], [C]~
+
+#+BEGIN_SRC markdown
+To make a NAS benchmark type
+
+      make <benchmark-name> CLASS=<class>
+
+where <benchmark-name> is "bt", "cg", "ep", "ft", "is", "lu",
+                          "lu-hp", "mg", "sp", or "ua"
+      <class>          is "S", "W", "A", "B", "C" or "D"
+#+END_SRC
+
+** pre-requirements
+
+#+begin_src bash
+sudo apt-get install -y openmpi-bin
+sudo apt-get install -y libopenmpi-dev
+sudo apt-get install -y fort77
+sudo apt-get install -y gfortran
+#+end_src

broker/test_setup/datasets.org

@@ -2,7 +2,6 @@
 
 #+begin_src bash
 b6aaf03752dc68d625fc57b451faa2bf  BL06-gargoyle-sml.tbz2
-f1de03edab51f281815c3c1e5ecb88c6  babyface.n6c100.tbz2
 082d2a71d86a64250f06be14c55ca27e  babyface.n6c10.tbz2
 03919732a417cb1d14049844b9de0f47  BL06-camel-med.tbz2
 983b9fe8a85b543dd5a4a75d031f1091  BL06-camel-sml.tbz2
@@ -17,8 +16,27 @@ ebe53bd498a9f6446cd77d9252a9847c  LB07-bunny-med.tbz2
 f82aa511f8631bfc9a82fe6fa30f4b52  LB07-bunny-sml.tbz2
 761691119cedfb9836a78a08742b14cc  liver.n6c100.tbz2
 f93b9a9f63447e0e086322b8416d4a39  liver.n6c10.tbz2
+
+#+end_src
+
+#+begin_src bash
+wget -bqc https://vision.cs.uwaterloo.ca/files/BL06-gargoyle-sml.tbz2
+wget -bqc https://vision.cs.uwaterloo.ca/files/babyface.n6c10.tbz2
+wget -bqc https://vision.cs.uwaterloo.ca/files/BL06-camel-sml.tbz2
+wget -bqc https://vision.cs.uwaterloo.ca/files/LB07-bunny-sml.tbz2
+wget -bqc https://vision.cs.uwaterloo.ca/files/BVZ-sawtooth.tbz2
+wget -bqc https://vision.cs.uwaterloo.ca/files/BVZ-tsukuba.tbz2
+wget -bqc https://vision.cs.uwaterloo.ca/files/BVZ-venus.tbz2
+wget -bqc https://vision.cs.uwaterloo.ca/files/KZ2-tsukuba.tbz2
+wget -bqc https://vision.cs.uwaterloo.ca/files/LB07-bunny-sml.tbz2
+wget -bqc https://vision.cs.uwaterloo.ca/files/KZ2-sawtooth.tbz2
+wget -bqc https://vision.cs.uwaterloo.ca/files/KZ2-venus.tbz2
+wget -bqc https://vision.cs.uwaterloo.ca/files/liver.n6c10.tbz2
+wget -bqc https://vision.cs.uwaterloo.ca/files/liver.n6c100.tbz2
 #+end_src
 
+-----------------------------------------------------------------------
+
 * Medium size data-sets
 
 #+begin_src bash
@@ -35,7 +53,7 @@ fa64e96bcee96dbc480a1495bddbf53c  LB07-bunny-lrg.tbz2
 1bfca57fe54bc46ba948023f754521d6  liver.n26c10.tbz2
 #+end_src
 
-* Download all use wget
+* Download all Medium data files using ~wget~
 
 #+begin_src bash
 wget -bqc https://vision.cs.uwaterloo.ca/files/adhead.n6c100.tbz2

broker/test_setup/nas/job_cppr.yaml → broker/test_setup/job_cppr.yaml

@@ -1,21 +1,21 @@
 config:
-    requester_address: '0x64b570f0e7c019dc750c4a75c33dca55bdc51845'
+    requester_address: '0x24056c57e3b0933d2fa7d83fb9667d0efdfae64d'
     provider_address: '0x29e613b04125c16db3f3613563bfdd0ba24cb629'
     source_code:
-        storage_id: ipfs_gpg
+        storage_id: eudat
         cache_type: public
         path: ~/test_eblocbroker/run_cppr
         storage_hours: 0
     data:
         data1:
-            hash: 050e6cc8dd7e889bf7874689f1e1ead6
+            hash: 0d6c3288ef71d89fb93734972d4eb903
         data2:
-            hash: 779745f315060d1bc0cd44b7266fb4da
+            hash: 4613abc322e8f2fdeae9a5dd10f17540
         data3:
             cache_type: public
-            path: /home/alper/test_eblocbroker/dataset_zip/small/BVZ-tsukuba
+            path: /home/alper/test_eblocbroker/dataset_zip/small/KZ2-tsukuba
             storage_hours: 1
-            storage_id: ipfs_gpg
+            storage_id: eudat
     data_transfer_out: 5
     jobs:
         job1:

broker/test_setup/nas/job_nas.yaml → broker/test_setup/job_nas.yaml

@@ -1,8 +1,8 @@
 config:
-    requester_address: '0xee516025bf14f7118aa9ced5eb2adacfd5827d14'
-    provider_address: '0x1926b36af775e1312fdebcc46303ecae50d945af'
+    requester_address: '0x77c0b42b5c358ff7c97e268794b9ff6a278a0f1e'
+    provider_address: '0x29e613b04125c16db3f3613563bfdd0ba24cb629'
     source_code:
-        storage_id: ipfs_gpg
+        storage_id: eudat
         cache_type: public
         path: ~/test_eblocbroker/NPB3.3-SER_source_code
         storage_hours: 0

broker/test_setup/register_data_files.py

@@ -25,15 +25,15 @@
 ]
 
 hashes_medium_1 = [
-    "050e6cc8dd7e889bf7874689f1e1ead6",  # A
-    "9d5d892a63b5758090258300a59eb389",  # A
-    "779745f315060d1bc0cd44b7266fb4da",  # A
+    "fe801973c5b22ef6861f2ea79dc1eb9c",  # A
+    "0d6c3288ef71d89fb93734972d4eb903",  # A
+    "4613abc322e8f2fdeae9a5dd10f17540",  # A
 ]
 
 hashes_medium_2 = [
-    "fe801973c5b22ef6861f2ea79dc1eb9c",  # B
-    "0d6c3288ef71d89fb93734972d4eb903",  # B
-    "4613abc322e8f2fdeae9a5dd10f17540",  # B
+    "050e6cc8dd7e889bf7874689f1e1ead6",  # B
+    "9d5d892a63b5758090258300a59eb389",  # B
+    "779745f315060d1bc0cd44b7266fb4da",  # B
     "dd0fbccccf7a198681ab838c67b68fbf",  # C
     "45281dfec4618e5d20570812dea38760",  # C
     "fa64e96bcee96dbc480a1495bddbf53c",  # C

broker/test_setup/nas/submit_jobs.py → broker/test_setup/submit_jobs.py

@@ -32,6 +32,7 @@
 _log.ll.LOG_FILENAME = Path.home() / ".ebloc-broker" / "test.log"
 
 benchmarks = ["nas", "cppr"]
+benchmarks = ["cppr"]
 storage_ids = ["eudat", "gdrive", "ipfs"]
 ipfs_ids = ["ipfs", "ipfs_gpg"]
 
@@ -44,7 +45,7 @@
 #     ipfs_ids = ["ipfs"]
 #     providers = ["0x29e613b04125c16db3f3613563bfdd0ba24cb629"]
 
-test_dir = Path.home() / "ebloc-broker" / "broker" / "test_setup" / "nas"
+test_dir = Path.home() / "ebloc-broker" / "broker" / "test_setup"
 nas_yaml_fn = test_dir / "job_nas.yaml"
 cppr_yam_fn = test_dir / "job_cppr.yaml"
 
@@ -85,14 +86,14 @@ def create_cppr_job_script(idx):
     # ]
     registered_data_hashes_medium = {}
     registered_data_hashes_medium[0] = [
-        "050e6cc8dd7e889bf7874689f1e1ead6",  # A
-        "9d5d892a63b5758090258300a59eb389",  # A
-        "779745f315060d1bc0cd44b7266fb4da",  # A
+        "fe801973c5b22ef6861f2ea79dc1eb9c",  # A
+        "0d6c3288ef71d89fb93734972d4eb903",  # A
+        "4613abc322e8f2fdeae9a5dd10f17540",  # A
     ]
     registered_data_hashes_medium[1] = [
-        "fe801973c5b22ef6861f2ea79dc1eb9c",  # B
-        "0d6c3288ef71d89fb93734972d4eb903",  # B
-        "4613abc322e8f2fdeae9a5dd10f17540",  # B
+        "050e6cc8dd7e889bf7874689f1e1ead6",  # B
+        "9d5d892a63b5758090258300a59eb389",  # B
+        "779745f315060d1bc0cd44b7266fb4da",  # B
     ]
     registered_data_hashes_medium[2] = [
         "dd0fbccccf7a198681ab838c67b68fbf",  # C
@@ -138,6 +139,7 @@ def create_cppr_job_script(idx):
     f.write("    echo $file >> output.log\n")
     f.write("    (/usr/bin/time -v cppr -a pr $file) >> output.log 2>&1\n")
     f.write("done\n")
+
     # adding cppr to run with data hashes
     f.write("DATA_HASH='change_folder_hash'\n")
     f.write("if [[ '$DATA_HASH' != 'change_folder_hash' ]]; then\n")
@@ -190,7 +192,7 @@ def create_nas_job_script(is_small=False):
 def mini_tests_submit(storage_ids, provider_address):
     is_pass = True
     required_confs = 0
-    yaml_fn = Path.home() / "ebloc-broker" / "broker" / "test_setup" / "nas" / "job_nas.yaml"
+    yaml_fn = Path.home() / "ebloc-broker" / "broker" / "test_setup" / "job_nas.yaml"
     yaml_cfg = Yaml(yaml_fn)
     yaml_cfg["config"]["provider_address"] = provider_address
     for storage_id in storage_ids:
@@ -283,13 +285,13 @@ def main():
         raise Exception("mongodb is not running in the background")
 
     counter = 0
-    for _ in range(60):
+    for _ in range(80):
         for _ in range(2):  # submitted as batch is faster
             run_job(counter)
             counter += 1
 
-        sleep_time = randint(200, 400)
-        countdown(sleep_time)
+        sleep_duration = randint(200, 400)
+        countdown(sleep_duration)
 
 
 if __name__ == "__main__":

docker-compose.yml

@@ -9,10 +9,10 @@ services:
     ports:
       - "4001:4001" # Public IPFS Port
     volumes:
-      - ./volumes/lib:/var/lib/slurmd
-      - ./volumes/spool:/var/spool/slurm
-      - ./volumes/log:/var/log/slurm
-      - ./volumes/db:/var/lib/mysql
+      - ~/docker/volumes/lib:/var/lib/slurmd
+      - ~/docker/volumes/spool:/var/spool/slurm
+      - ~/docker/volumes/log:/var/log/slurm
+      - ~/docker/volumes/db:/var/lib/mysql
   # slurm:
   #   build:
   #     dockerfile: Dockerfile