diff --git a/.changeset/quiet-tables-walk.md b/.changeset/quiet-tables-walk.md
new file mode 100644
index 0000000..ec4189d
--- /dev/null
+++ b/.changeset/quiet-tables-walk.md
@@ -0,0 +1,5 @@
+---
+"smart-whisper": patch
+---
+
+Fix confidence algorithm
diff --git a/src/binding/transcribe.cc b/src/binding/transcribe.cc
index 6f0d464..dfe676e 100644
--- a/src/binding/transcribe.cc
+++ b/src/binding/transcribe.cc
@@ -253,9 +253,9 @@ class TranscribeWorker : public Napi::AsyncProgressQueueWorker<int> {
                     max_p = std::max(max_p, token.p);
                 }
 
-                if (tokens > 2) {
-                    confidence = (confidence - min_p - max_p) / (tokens - 2 - skips);
-                } else {
+                if (tokens - skips > 2) {
+                    confidence = (confidence - min_p - max_p) / (tokens - skips - 2);
+                } else if (tokens - skips > 0) {
                     confidence = confidence / (tokens - skips);
                 }