LLMs: Anthropic: add the Thinking variant

enricoros · Feb 25, 2025 · e428683 · e428683
1 parent b646222
commit e428683
Show file tree

Hide file tree

Showing 2 changed files with 48 additions and 17 deletions.
diff --git a/src/modules/llms/server/anthropic/anthropic.models.ts b/src/modules/llms/server/anthropic/anthropic.models.ts
@@ -3,6 +3,21 @@ import { LLM_IF_ANT_PromptCaching, LLM_IF_OAI_Chat, LLM_IF_OAI_Fn, LLM_IF_OAI_Vi
 import type { ModelDescriptionSchema } from '../llm.server.types';
 
 
+export const hardcodedAnthropicVariants: { [modelId: string]: Partial<ModelDescriptionSchema> } = {
+
+  // Changes to the thinking variant (same model ID) for the Claude 3.7 Sonnet model
+  'claude-3-7-sonnet-20250219': {
+    idVariant: 'thinking',
+    label: 'Claude 3.7 Sonnet (Thinking)',
+    description: 'Claude 3.7 with extended thinking mode enabled for complex reasoning',
+    parameterSpecs: [{ paramId: 'llmVndAntThinkingBudget', required: true, hidden: false }],
+    maxCompletionTokens: 65536, // Extended thinking mode - note that the 'anthropic-beta: output-128k-2025-02-19' header would point to a 128k instead
+    benchmark: { cbaElo: 1283 + 1 },
+  },
+
+} as const;
+
+
 export const hardcodedAnthropicModels: (ModelDescriptionSchema & { isLegacy?: boolean })[] = [
 
   // Claude 3.7 models
@@ -11,11 +26,11 @@ export const hardcodedAnthropicModels: (ModelDescriptionSchema & { isLegacy?: bo
     label: 'Claude 3.7 Sonnet', // 🌟
     description: 'Highest level of intelligence and capability with toggleable extended thinking',
     contextWindow: 200000,
-    maxCompletionTokens: 64000, // Extended thinking mode - note that the 'anthropic-beta: output-128k-2025-02-19' header would point to a 128k instead
+    maxCompletionTokens: 8192,
     trainingDataCutoff: 'Oct 2024',
     interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_ANT_PromptCaching],
     chatPrice: { input: 3, output: 15, cache: { cType: 'ant-bp', read: 0.30, write: 3.75, duration: 300 } },
-    benchmark: { cbaElo: 1283 + 1 /* temporary to be preferred/on top */ },
+    benchmark: { cbaElo: 1283 + 2 /* temporary to be preferred/on top */ },
   },
 
   // Claude 3.5 models

diff --git a/src/modules/llms/server/anthropic/anthropic.router.ts b/src/modules/llms/server/anthropic/anthropic.router.ts
@@ -9,7 +9,7 @@ import { fixupHost } from '~/common/util/urlUtils';
 
 import { ListModelsResponse_schema, ModelDescriptionSchema } from '../llm.server.types';
 
-import { hardcodedAnthropicModels } from './anthropic.models';
+import { hardcodedAnthropicModels, hardcodedAnthropicVariants } from './anthropic.models';
 
 
 // configuration and defaults
@@ -160,22 +160,38 @@ export const llmAnthropicRouter = createTRPCRouter({
       const { data: availableModels } = AnthropicWire_API_Models_List.Response_schema.parse(wireModels);
 
       // cast the models to the common schema
-      const models = availableModels.map((model): ModelDescriptionSchema => {
+      const models = availableModels.reduce((acc, model) => {
 
-        // use an hardcoded model definition if available
+        // find the model description
         const hardcodedModel = hardcodedAnthropicModels.find(m => m.id === model.id);
-        if (hardcodedModel && !hardcodedModel.created && model.created_at)
-          hardcodedModel.created = roundTime(model.created_at);
-        if (hardcodedModel)
-          return hardcodedModel;
-
-        // for day-0 support of new models, create a placeholder model using sensible defaults
-        const novelModel = _createPlaceholderModel(model);
-        console.log('[DEV] anthropic.router: new model found, please configure it:', novelModel.id);
-        return novelModel;
-      });
-
-      // developers warning for obsoleted models
+        if (hardcodedModel) {
+
+          // update creation date
+          if (!hardcodedModel.created && model.created_at)
+            hardcodedModel.created = roundTime(model.created_at);
+
+          // add the base model
+          acc.push(hardcodedModel);
+
+          // add a thinking variant, if defined
+          if (hardcodedAnthropicVariants[model.id])
+            acc.push({
+              ...hardcodedModel,
+              ...hardcodedAnthropicVariants[model.id],
+            });
+
+        } else {
+
+          // for day-0 support of new models, create a placeholder model using sensible defaults
+          const novelModel = _createPlaceholderModel(model);
+          console.log('[DEV] anthropic.router: new model found, please configure it:', novelModel.id);
+          acc.push(novelModel);
+
+        }
+        return acc;
+      }, [] as ModelDescriptionSchema[]);
+
+      // developers warning for obsoleted models (we have them, but they are not in the API response anymore)
       const apiModelIds = new Set(availableModels.map(m => m.id));
       const additionalModels = hardcodedAnthropicModels.filter(m => !apiModelIds.has(m.id));
       if (additionalModels.length > 0)