Merge branch 'develop' into deadline_timer_cleanup

.github/workflows/analyzers.yml

@@ -5,6 +5,7 @@ on: [push, pull_request]
 jobs:
   clang_format:
     runs-on: ubuntu-18.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - uses: actions/checkout@50fbc62
       - name: Get clang-format 8

.github/workflows/beta_artifacts.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Fetch Deps
         run: ci/actions/linux/install_deps.sh
       - name: Build Artifact
-        run: docker run -v ${GITHUB_WORKSPACE}:/workspace nanocurrency/nano-env:gcc /bin/bash -c "cd /workspace && BETA=1 TRAVIS_TAG=${TAG} ci/build-deploy.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"
+        run: docker run -v ${GITHUB_WORKSPACE}:/workspace nanocurrency/nano-env:gcc /bin/bash -c "cd /workspace && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp/boost/lib && ldconfig && BETA=1 TRAVIS_TAG=${TAG} ci/build-deploy.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"
       - name: Deploy Artifact
         run: ci/actions/deploy.sh
         env:

.github/workflows/live_artifacts.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Fetch Deps
         run: ci/actions/linux/install_deps.sh
       - name: Build Artifact
-        run: docker run -v ${GITHUB_WORKSPACE}:/workspace nanocurrency/nano-env:gcc /bin/bash -c "cd /workspace && TRAVIS_TAG=${TAG} ci/build-deploy.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"
+        run: docker run -v ${GITHUB_WORKSPACE}:/workspace nanocurrency/nano-env:gcc /bin/bash -c "cd /workspace && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp/boost/lib && ldconfig && TRAVIS_TAG=${TAG} ci/build-deploy.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"
       - name: Deploy Artifact
         run: ci/actions/deploy.sh
         env:

.github/workflows/rocksdb_release_tests.yml

@@ -0,0 +1,48 @@
+name: RocksDB Release Tests
+
+on: 
+  push: 
+    tags: '*'
+
+env:
+  RELEASE: 1
+  artifact: 0
+  TEST_USE_ROCKSDB: 1
+
+jobs:
+  osx_test:
+    runs-on: macOS-latest
+    env:
+      BOOST_ROOT: /tmp/boost
+    steps:
+      - uses: actions/checkout@722adc6
+      - name: Checkout Submodules
+        run: git submodule update --init --recursive
+      - name: Fetch Deps
+        run: TEST=1 ci/actions/osx/install_deps.sh
+      - name: Run Tests
+        run: ci/build-travis.sh "/tmp/qt/lib/cmake/Qt5";
+
+  gcc_test:
+    runs-on: ubuntu-18.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@722adc6
+      - name: Checkout Submodules
+        run: git submodule update --init --recursive
+      - name: Fetch Deps
+        run: ci/actions/linux/install_deps.sh
+      - name: Run Tests
+        run: docker run -e TEST_USE_ROCKSDB -v ${PWD}:/workspace nanocurrency/nano-env:gcc /bin/bash -c "cd /workspace && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp/boost/lib && ldconfig && RELEASE=1 ./ci/build-travis.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"
+
+  clang_test:
+    runs-on: ubuntu-18.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@722adc6
+      - name: Checkout Submodules
+        run: git submodule update --init --recursive
+      - name: Fetch Deps
+        run: ci/actions/linux/install_deps.sh
+      - name: Run Tests
+        run: docker run -e TEST_USE_ROCKSDB -v ${PWD}:/workspace nanocurrency/nano-env:clang-6 /bin/bash -c "cd /workspace && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp/boost/lib && ldconfig && RELEASE=1 ./ci/build-travis.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"

.github/workflows/rocksdb_tests.yml

@@ -0,0 +1,68 @@
+name: RocksDB Tests
+
+on: [push, pull_request]
+
+env:
+  RELEASE: 0
+  artifact: 0
+  TEST_USE_ROCKSDB: 1
+
+jobs:
+  osx_test:
+    runs-on: macOS-latest
+    env:
+      BOOST_ROOT: /tmp/boost
+      DEADLINE_SCALE_FACTOR: 2
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - uses: actions/checkout@722adc6
+      - name: Checkout Submodules
+        run: git submodule update --init --recursive
+      - name: Fetch Deps
+        run: TEST=1 ci/actions/osx/install_deps.sh
+      - name: Run Tests
+        run: ci/build-travis.sh "/tmp/qt/lib/cmake/Qt5";
+
+  gcc_test:
+    runs-on: ubuntu-18.04
+    timeout-minutes: 60
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - uses: actions/checkout@722adc6
+      - name: Checkout Submodules
+        run: git submodule update --init --recursive
+      - name: Fetch Deps
+        run: ci/actions/linux/install_deps.sh
+      - name: Run Tests
+        run: docker run -e TEST_USE_ROCKSDB -v ${PWD}:/workspace nanocurrency/nano-env:gcc /bin/bash -c "cd /workspace && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp/boost/lib && ldconfig && ./ci/build-travis.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"
+
+  clang_test:
+    runs-on: ubuntu-18.04
+    timeout-minutes: 60
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - uses: actions/checkout@722adc6
+      - name: Checkout Submodules
+        run: git submodule update --init --recursive
+      - name: Fetch Deps
+        run: ci/actions/linux/install_deps.sh
+      - name: Run Tests
+        run: docker run -e TEST_USE_ROCKSDB -v ${PWD}:/workspace nanocurrency/nano-env:clang-6 /bin/bash -c "cd /workspace && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp/boost/lib && ldconfig && ./ci/build-travis.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"
+
+  windows_test:
+    runs-on: windows-latest
+    timeout-minutes: 60
+    env:
+      RELEASE: 1
+      DEADLINE_SCALE_FACTOR: 2
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - uses: actions/checkout@722adc6
+      - name: Windows Defender
+        run: ci/actions/windows/disable_windows_defender.ps1
+      - name: Checkout Submodules
+        run: git submodule update --init --recursive
+      - name: Fetch Deps
+        run: ci/actions/windows/install_deps.ps1
+      - name: Run Tests
+        run: ci/actions/windows/build.ps1

.github/workflows/tests.yml

@@ -11,6 +11,7 @@ jobs:
     runs-on: macOS-latest
     env:
       BOOST_ROOT: /tmp/boost
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - uses: actions/checkout@722adc6
       - name: Checkout Submodules
@@ -23,6 +24,7 @@ jobs:
   gcc_test:
     runs-on: ubuntu-18.04
     timeout-minutes: 60
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - uses: actions/checkout@722adc6
       - name: Checkout Submodules
@@ -35,6 +37,7 @@ jobs:
   clang_test:
     runs-on: ubuntu-18.04
     timeout-minutes: 60
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - uses: actions/checkout@722adc6
       - name: Checkout Submodules
@@ -47,6 +50,7 @@ jobs:
   windows_test:
     runs-on: windows-latest
     timeout-minutes: 60
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - uses: actions/checkout@722adc6
       - name: Windows Defender

.gitmodules

@@ -24,3 +24,7 @@
 [submodule "flatbuffers"]
 	path = flatbuffers
 	url = https://github.com/google/flatbuffers.git
+[submodule "rocksdb"]
+	path = rocksdb
+	url = https://github.com/nanocurrency/rocksdb.git
+	branch = 6.11.4

CMakeLists.txt

@@ -16,7 +16,7 @@ endif ()
 
 # compatibility for osx sierra and on
 # needs to be set before project
-set (CMAKE_OSX_DEPLOYMENT_TARGET 10.12 CACHE STRING "")
+set (CMAKE_OSX_DEPLOYMENT_TARGET 10.14 CACHE STRING "")
 
 project (nano-node)
 
@@ -59,13 +59,13 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 set (NANO_GUI OFF CACHE BOOL "")
 set (NANO_TEST OFF CACHE BOOL "")
 set (NANO_SECURE_RPC OFF CACHE BOOL "")
-set (NANO_ROCKSDB OFF CACHE BOOL "")
 set (NANO_POW_SERVER OFF CACHE BOOL "")
 set (NANO_WARN_TO_ERR OFF CACHE BOOL "")
 set (NANO_TIMED_LOCKS 0 CACHE STRING "")
 set (NANO_TIMED_LOCKS_IGNORE_BLOCKED OFF CACHE BOOL "")
 set (NANO_FUZZER_TEST OFF CACHE BOOL "")
 set (NANO_ASIO_HANDLER_TRACKING 0 CACHE STRING "")
+set (NANO_ROCKSDB_TOOLS OFF CACHE BOOL "")
 
 option (NANO_STACKTRACE_BACKTRACE "Use BOOST_STACKTRACE_USE_BACKTRACE in stacktraces, for POSIX" OFF)
 if (NANO_STACKTRACE_BACKTRACE)
@@ -86,8 +86,6 @@ if (${NANO_ASIO_HANDLER_TRACKING} GREATER 0)
 	add_definitions (-DNANO_ASIO_HANDLER_TRACKING=${NANO_ASIO_HANDLER_TRACKING} -DBOOST_ASIO_ENABLE_HANDLER_TRACKING)
 endif ()
 
-add_definitions (-DNANO_ROCKSDB=$<STREQUAL:${NANO_ROCKSDB},ON>)
-
 option(NANO_ASAN_INT "Enable ASan+UBSan+Integer overflow" OFF)
 option(NANO_ASAN "Enable ASan+UBSan" OFF)
 option(NANO_TSAN "Enable TSan" OFF)
@@ -252,21 +250,31 @@ if (NANO_SHARED_BOOST)
 	SET(Boost_USE_STATIC_LIBS OFF)
 	SET(Boost_USE_STATIC_RUNTIME OFF)
 	SET(Boost_NO_BOOST_CMAKE ON)
-	add_definitions( -DBOOST_ALL_DYN_LINK -DBoost_ALL_NO_LIB)
+	add_definitions(-DBOOST_ALL_DYN_LINK -DBoost_ALL_NO_LIB)
 else()
 	set(Boost_USE_STATIC_LIBS ON)
 endif()
 set(Boost_USE_MULTITHREADED ON)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules")
 
-find_package (Boost 1.69.0 REQUIRED COMPONENTS filesystem log log_setup thread program_options system)
-
-if (NANO_ROCKSDB)
-	find_package (RocksDB REQUIRED)
-	find_package (ZLIB REQUIRED)
-	include_directories (${ROCKSDB_INCLUDE_DIRS})
-endif ()
+find_package (Boost 1.69.0 REQUIRED COMPONENTS coroutine context filesystem log log_setup thread program_options system)
+
+# RocksDB
+include_directories(rocksdb/include)
+set(USE_RTTI ON CACHE BOOL "")
+set(WITH_GFLAGS OFF CACHE BOOL "")
+set(WITH_TESTS OFF CACHE BOOL "")
+set(WITH_BENCHMARK_TOOLS OFF CACHE BOOL "")
+set(ROCKSDB_BUILD_SHARED OFF CACHE BOOL "")
+set(WITH_CORE_TOOLS ${NANO_ROCKSDB_TOOLS} CACHE BOOL "" FORCE)
+set(WITH_TOOLS ${NANO_ROCKSDB_TOOLS} CACHE BOOL "" FORCE)
+if (ENABLE_AVX2)
+	set(PORTABLE OFF CACHE BOOL "" FORCE)
+else()
+	set(PORTABLE ON CACHE BOOL "" FORCE)
+endif()
+add_subdirectory(rocksdb)
 
 # There is a compile bug with boost 1.69 interprocess headers on Mac
 if (APPLE AND Boost_VERSION EQUAL 106900)
@@ -426,8 +434,6 @@ if (NANO_FUZZER_TEST)
 endif ()
 
 if (NANO_TEST OR RAIBLOCKS_TEST)
-	find_package (Boost 1.69.0 REQUIRED COMPONENTS coroutine)
-
 	if(WIN32)
 		if(MSVC_VERSION)
 			if(MSVC_VERSION GREATER_EQUAL 1910)

ci/actions/osx/install_deps.sh

@@ -3,10 +3,5 @@
 brew update
 brew install coreutils
 brew cask install xquartz
-if [[ ${TEST-0} -eq 1 ]]; then
-    brew install rocksdb;
-else
-    util/build_prep/fetch_rocksdb.sh
-fi
 sudo util/build_prep/fetch_boost.sh 
 util/build_prep/macosx/build_qt.sh

ci/actions/windows/configure.bat

@@ -8,12 +8,8 @@ cmake .. ^
   -Ax64 ^
   %NANO_TEST% ^
   %CI% ^
-  -DNANO_ROCKSDB=ON ^
   %ROCKS_LIB% ^
-  -DROCKSDB_INCLUDE_DIRS="c:\vcpkg\installed\x64-windows-static\include" ^
-  -DZLIB_LIBRARY_RELEASE="c:\vcpkg\installed\x64-windows-static\lib\zlib.lib" ^
-  -DZLIB_LIBRARY_DEBUG="c:\vcpkg\installed\x64-windows-static\debug\lib\zlibd.lib" ^
-  -DZLIB_INCLUDE_DIR="c:\vcpkg\installed\x64-windows-static\include" ^
+  -DPORTABLE=1 ^
   -DQt5_DIR="c:\qt\5.13.1\msvc2017_64\lib\cmake\Qt5" ^
   -DNANO_GUI=ON ^
   -DCMAKE_BUILD_TYPE=%BUILD_TYPE% ^

ci/actions/windows/install_deps.ps1

@@ -50,7 +50,7 @@ function Get-RedirectedUri {
 }
 
 $qt5_root = "c:\qt"
-$rocksdb_url = Get-RedirectedUri "https://repo.nano.org/artifacts/rocksdb-msvc14.1-latest.7z"
+$rocksdb_url = Get-RedirectedUri "https://repo.nano.org/artifacts/rocksdb-msvc-14.2-6.6.4-Md.7z"
 $qt5base_url = Get-RedirectedUri "https://repo.nano.org/artifacts/5.13.1-0-201909031231qtbase-Windows-Windows_10-MSVC2017-Windows-Windows_10-X86_64.7z"
 $qt5winextra_url = Get-RedirectedUri "https://repo.nano.org/artifacts/5.13.1-0-201909031231qtwinextras-Windows-Windows_10-MSVC2017-Windows-Windows_10-X86_64.7z"
 $rocksdb_artifact = "${env:TMP}\rocksdb.7z"
@@ -61,10 +61,10 @@ $qt5winextra_artifact = "${env:TMP}\qt5winextra.7z"
 (New-Object System.Net.WebClient).DownloadFile($qt5winextra_url, $qt5winextra_artifact)
 mkdir $qt5_root
 Push-Location $qt5_root
-7z x "${env:TMP}\qt5*.7z"
+7z x "${env:TMP}\qt5*.7z" -aoa
 Pop-Location
 
 Push-Location ${env:VCPKG_INSTALLATION_ROOT} 
 (New-Object System.Net.WebClient).DownloadFile($rocksdb_url, $rocksdb_artifact)
-7z x $rocksdb_artifact
+7z x $rocksdb_artifact -aoa
 Pop-Location 

ci/build-deploy.sh

@@ -24,9 +24,7 @@ cmake \
     -DACTIVE_NETWORK=nano_${NETWORK_CFG}_network \
     -DNANO_POW_SERVER=ON \
     -DNANO_GUI=ON \
-    -DNANO_ROCKSDB=ON \
-    -DROCKSDB_LIBRARIES=/tmp/rocksdb/lib/librocksdb.a \
-    -DROCKSDB_INCLUDE_DIRS=/tmp/rocksdb/include \
+    -DPORTABLE=1 \
     -DCMAKE_BUILD_TYPE=${CONFIGURATION} \
     -DCMAKE_VERBOSE_MAKEFILE=ON \
     -DBOOST_ROOT=/tmp/boost/ \

ci/build-travis.sh

@@ -48,16 +48,13 @@ fi
 ulimit -S -n 8192
 
 if [[ "$OS" == 'Linux' ]]; then
-    ROCKSDB="-DROCKSDB_LIBRARIES=/tmp/rocksdb/lib/librocksdb.a \
-    -DROCKSDB_INCLUDE_DIRS=/tmp/rocksdb/include"
     if clang --version; then
         BACKTRACE="-DNANO_STACKTRACE_BACKTRACE=ON \
         -DBACKTRACE_INCLUDE=</tmp/backtrace.h>"
     else
         BACKTRACE="-DNANO_STACKTRACE_BACKTRACE=ON"
     fi
 else
-    ROCKSDB=""
     BACKTRACE=""
 fi
 
@@ -66,8 +63,7 @@ cmake \
     -DACTIVE_NETWORK=nano_dev_network \
     -DNANO_TEST=ON \
     -DNANO_GUI=ON \
-    -DNANO_ROCKSDB=ON \
-    ${ROCKSDB} \
+    -DPORTABLE=1 \
     -DNANO_WARN_TO_ERR=ON \
     -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
     -DCMAKE_VERBOSE_MAKEFILE=ON \

ci/test.sh

@@ -1,7 +1,11 @@
 #!/usr/bin/env bash
 
 build_dir=${1-${PWD}}
-TIMEOUT_DEFAULT=360
+if [[ ${TEST_USE_ROCKSDB-0} == 1 ]]; then
+    TIMEOUT_DEFAULT=720
+else
+    TIMEOUT_DEFAULT=360
+fi
 
 BUSYBOX_BASH=${BUSYBOX_BASH-0}