diff --git a/packages/nx/src/tasks-runner/cache.ts b/packages/nx/src/tasks-runner/cache.ts
index fe6ff3aaea1b1..648ca6cb4ad1f 100644
--- a/packages/nx/src/tasks-runner/cache.ts
+++ b/packages/nx/src/tasks-runner/cache.ts
@@ -276,7 +276,7 @@ export class Cache {
 
   private tryAndRetry<T>(fn: () => Promise<T>): Promise<T> {
     let attempts = 0;
-    const baseTimeout = 100;
+    const baseTimeout = 5;
     // Generate a random number between 2 and 4 to raise to the power of attempts
     const baseExponent = Math.random() * 2 + 2;
     const _try = async () => {
@@ -284,13 +284,12 @@ export class Cache {
         attempts++;
         return await fn();
       } catch (e) {
-        if (attempts === 10) {
+        // Max time is 5 * 4^3 = 20480ms
+        if (attempts === 6) {
           // After enough attempts, throw the error
           throw e;
         }
-        await new Promise((res) =>
-          setTimeout(res, baseTimeout * baseExponent ** attempts)
-        );
+        await new Promise((res) => setTimeout(res, baseExponent ** attempts));
         return await _try();
       }
     };