Correct other models as well

dqbd · Feb 4, 2025 · 9fc37ed · 9fc37ed
1 parent 4dbb38e
commit 9fc37ed
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 4 deletions.
diff --git a/tiktoken/model_to_encoding.json b/tiktoken/model_to_encoding.json
@@ -57,8 +57,8 @@
     "gpt-4-vision-preview": "cl100k_base",
     "gpt-4o": "o200k_base",
     "gpt-4o-2024-05-13": "o200k_base",
-    "gpt-4o-2024-08-06":"o200k_base",
-    "gpt-4o-2024-11-20":"o200k_base",
+    "gpt-4o-2024-08-06": "o200k_base",
+    "gpt-4o-2024-11-20": "o200k_base",
     "gpt-4o-mini-2024-07-18": "o200k_base",
     "gpt-4o-mini": "o200k_base",
     "o1": "o200k_base",

diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs
@@ -460,9 +460,10 @@ pub fn encoding_for_model(
         "text-babbage-001" => Ok("r50k_base"),
         "text-ada-001" => Ok("r50k_base"),
         "davinci" => Ok("r50k_base"),
+        "davinci-002" => Ok("cl100k_base"),
         "curie" => Ok("r50k_base"),
         "babbage" => Ok("r50k_base"),
-        "babbage-002" => Ok("r50k_base"),
+        "babbage-002" => Ok("cl100k_base"),
         "ada" => Ok("r50k_base"),
         "code-davinci-002" => Ok("p50k_base"),
         "code-davinci-001" => Ok("p50k_base"),
@@ -491,7 +492,7 @@ pub fn encoding_for_model(
         "gpt-3.5-turbo-0613" => Ok("cl100k_base"),
         "gpt-3.5-turbo-16k" => Ok("cl100k_base"),
         "gpt-3.5-turbo-16k-0613" => Ok("cl100k_base"),
-        "gpt-3.5-turbo-instruct" => Ok("clk100k_base"),
+        "gpt-3.5-turbo-instruct" => Ok("cl100k_base"),
         "gpt-3.5-turbo-instruct-0914" => Ok("cl100k_base"),
         "gpt-4" => Ok("cl100k_base"),
         "gpt-4-0314" => Ok("cl100k_base"),
@@ -514,6 +515,7 @@ pub fn encoding_for_model(
         "gpt-4o-2024-11-20" => Ok("o200k_base"),
         "gpt-4o-mini-2024-07-18" => Ok("o200k_base"),
         "gpt-4o-mini" => Ok("o200k_base"),
+        "o1" => Ok("o200k_base"),
         "o1-2024-12-17" => Ok("o200k_base"),
         "o1-mini" => Ok("o200k_base"),
         "o1-preview" => Ok("o200k_base"),