Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat 59 - Tesseract support #71

Merged
merged 5 commits into from
Aug 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file not shown.
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
"sonner": "^1.5.0",
"tailwind-merge": "^2.3.0",
"tailwindcss-animate": "^1.0.7",
"tesseract.js": "^5.1.0",
"typescript-string-operations": "^1.5.1",
"uuid": "^10.0.0",
"vaul": "^0.9.1",
Expand Down
2 changes: 1 addition & 1 deletion src/components/patient-record-item.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ export default function PatientRecordItem({ record, displayAttachmentPreviews }:
useEffect(() => {
if (displayAttachmentPreviews && !displayableAttachmentsInProgress) {
displayableAttachmentsInProgress = true;
patientRecordContext?.convertAttachmentsToImages(record).then((attachments) => {
patientRecordContext?.convertAttachmentsToImages(record, false).then((attachments) => {
setDisplayableAttachments(attachments);
displayableAttachmentsInProgress = false;
});
Expand Down
215 changes: 206 additions & 9 deletions src/components/settings-popup.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -11,31 +11,161 @@ import { ConfigContext } from "@/contexts/config-context"
import React from "react"
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "./ui/card";
import { useForm } from "react-hook-form";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "./ui/select";
import { set } from "react-hook-form";

const ocrLlanguages = [
{ name: "English", code: "eng" },
{ name: "Polish", code: "pol" },
{ name: "Portuguese", code: "por" },
{ name: "Afrikaans", code: "afr" },
{ name: "Albanian", code: "sqi" },
{ name: "Amharic", code: "amh" },
{ name: "Arabic", code: "ara" },
{ name: "Assamese", code: "asm" },
{ name: "Azerbaijani", code: "aze" },
{ name: "Azerbaijani - Cyrillic", code: "aze_cyrl" },
{ name: "Basque", code: "eus" },
{ name: "Belarusian", code: "bel" },
{ name: "Bengali", code: "ben" },
{ name: "Bosnian", code: "bos" },
{ name: "Bulgarian", code: "bul" },
{ name: "Burmese", code: "mya" },
{ name: "Catalan; Valencian", code: "cat" },
{ name: "Cebuano", code: "ceb" },
{ name: "Central Khmer", code: "khm" },
{ name: "Cherokee", code: "chr" },
{ name: "Chinese - Simplified", code: "chi_sim" },
{ name: "Chinese - Traditional", code: "chi_tra" },
{ name: "Croatian", code: "hrv" },
{ name: "Czech", code: "ces" },
{ name: "Danish", code: "dan" },
{ name: "Dutch; Flemish", code: "nld" },
{ name: "Dzongkha", code: "dzo" },
{ name: "English, Middle (1100-1500)", code: "enm" },
{ name: "Esperanto", code: "epo" },
{ name: "Estonian", code: "est" },
{ name: "Finnish", code: "fin" },
{ name: "French", code: "fra" },
{ name: "French, Middle (ca. 1400-1600)", code: "frm" },
{ name: "Galician", code: "glg" },
{ name: "Georgian", code: "kat" },
{ name: "German", code: "deu" },
{ name: "German Fraktur", code: "frk" },
{ name: "Greek, Modern (1453-)", code: "ell" },
{ name: "Greek, Ancient (-1453)", code: "grc" },
{ name: "Gujarati", code: "guj" },
{ name: "Haitian; Haitian Creole", code: "hat" },
{ name: "Hebrew", code: "heb" },
{ name: "Hindi", code: "hin" },
{ name: "Hungarian", code: "hun" },
{ name: "Icelandic", code: "isl" },
{ name: "Indonesian", code: "ind" },
{ name: "Inuktitut", code: "iku" },
{ name: "Irish", code: "gle" },
{ name: "Italian", code: "ita" },
{ name: "Japanese", code: "jpn" },
{ name: "Javanese", code: "jav" },
{ name: "Kannada", code: "kan" },
{ name: "Kazakh", code: "kaz" },
{ name: "Kirghiz; Kyrgyz", code: "kir" },
{ name: "Korean", code: "kor" },
{ name: "Kurdish", code: "kur" },
{ name: "Lao", code: "lao" },
{ name: "Latin", code: "lat" },
{ name: "Latvian", code: "lav" },
{ name: "Lithuanian", code: "lit" },
{ name: "Macedonian", code: "mkd" },
{ name: "Malay", code: "msa" },
{ name: "Malayalam", code: "mal" },
{ name: "Maltese", code: "mlt" },
{ name: "Marathi", code: "mar" },
{ name: "Nepali", code: "nep" },
{ name: "Norwegian", code: "nor" },
{ name: "Oriya", code: "ori" },
{ name: "Panjabi; Punjabi", code: "pan" },
{ name: "Persian", code: "fas" },
{ name: "Pushto; Pashto", code: "pus" },
{ name: "Romanian; Moldavian; Moldovan", code: "ron" },
{ name: "Russian", code: "rus" },
{ name: "Sanskrit", code: "san" },
{ name: "Serbian", code: "srp" },
{ name: "Serbian - Latin", code: "srp_latn" },
{ name: "Sinhala; Sinhalese", code: "sin" },
{ name: "Slovak", code: "slk" },
{ name: "Slovenian", code: "slv" },
{ name: "Spanish; Castilian", code: "spa" },
{ name: "Swahili", code: "swa" },
{ name: "Swedish", code: "swe" },
{ name: "Syriac", code: "syr" },
{ name: "Tagalog", code: "tgl" },
{ name: "Tajik", code: "tgk" },
{ name: "Tamil", code: "tam" },
{ name: "Telugu", code: "tel" },
{ name: "Thai", code: "tha" },
{ name: "Tibetan", code: "bod" },
{ name: "Tigrinya", code: "tir" },
{ name: "Turkish", code: "tur" },
{ name: "Uighur; Uyghur", code: "uig" },
{ name: "Ukrainian", code: "ukr" },
{ name: "Urdu", code: "urd" },
{ name: "Uzbek", code: "uzb" },
{ name: "Uzbek - Cyrillic", code: "uzb_cyrl" },
{ name: "Vietnamese", code: "vie" },
{ name: "Welsh", code: "cym" },
{ name: "Yiddish", code: "yid" },
];

export function SettingsPopup() {
const config = useContext(ConfigContext);

const [ocrProvider, setOcrProvider] = useState("chatgpt");
const [ocrLanguage, setOcrLanguage] = useState("eng");
const [llmProvider, setLlmProvider] = useState("chatgpt")
const [llmMode, setLlmMode] = useState("both")

const { handleSubmit, register, setError, getValues, setValue, formState: { errors, } } = useForm({
defaultValues: {
chatGptApiKey: "",
displayAttachmentPreviews: true
displayAttachmentPreviews: true,
ocrProvider: "chatgpt",
ocrLanguage: "eng",
ollamaUrl: ""

}
});

useEffect(() => { // load default configuration
async function fetchDefaultConfig() {
const chatGptKey = await config?.getServerConfig('chatGptApiKey');
const displayAttachmentPreviews = await config?.getServerConfig('displayAttachmentPreviews');
const ocr = await config?.getServerConfig('ocrProvider') as string
const ocrLang = await config?.getServerConfig('ocrLanguage') as string
const ollamaUrl = await config?.getServerConfig('ollamaUrl') as string
const llmMode = await config?.getServerConfig('llmMode') as string
const llmProvider = await config?.getServerConfig('llmProvider') as string;
setOcrProvider(ocr);
setOcrLanguage(ocrLang);
setLlmProvider(llmProvider);
setLlmMode(llmMode);

setValue("chatGptApiKey", chatGptKey as string);
setValue("displayAttachmentPreviews", displayAttachmentPreviews as boolean);
setValue("ocrProvider", ocr);
setValue("ocrLanguage", "eng");
setValue("llmProvider", "chatgpt");
setValue("ollamaUrl", ollamaUrl);
}
fetchDefaultConfig();
}, []);

async function onSubmit(formData) {
async function onSubmit(formData) { // TODO: we probably need a method in the config-context to setup the config model - so all available server and local config variables with default values
config?.setServerConfig('chatGptApiKey', formData['chatGptApiKey']);
config?.setServerConfig('ocrProvider', ocrProvider as string);
config?.setServerConfig('displayAttachmentPreviews', formData['displayAttachmentPreviews']);
config?.setServerConfig('ocrLanguage', ocrLanguage as string);
config?.setServerConfig('llmProvider', llmProvider as string);
config?.setServerConfig('ollamaUrl', formData['ollamaUrl']);
config?.setServerConfig('llmMode', llmMode as string);
config?.setConfigDialogOpen(false);
}

Expand Down Expand Up @@ -76,12 +206,79 @@ export function SettingsPopup() {
</TabsContent>
<TabsContent value="ai-settings">
<Card>
<CardHeader>
<CardTitle>Settings</CardTitle>
<CardDescription>
Setup AI related settings here
</CardDescription>
</CardHeader>
<CardHeader>Local AI Models & OCR</CardHeader>
<CardContent className="space-y-2">
<div className="grid grid-cols-1 gap-3">
<div className="grid gap-2">
<div className="flex items-center gap-2">
<Label htmlFor="ocrProvider">OCR Provider</Label>
<Select id="ocrProvider" value={ocrProvider} onValueChange={setOcrProvider}>
<SelectTrigger className="w-[180px]">
<SelectValue placeholder="Default: Chat GPT" />
</SelectTrigger>
<SelectContent>
<SelectItem key="chatgpt" value="chatgpt">Default: Chat GPT</SelectItem>
<SelectItem key="tesseract" value="tesseract">Tesseract</SelectItem>
</SelectContent>
</Select>
</div>
<div className="flex items-center gap-2">
<Label htmlFor="ocrLanguage">OCR Language</Label>
<Select id="ocrLanguage" value={ocrLanguage} onValueChange={setOcrLanguage}>
<SelectTrigger className="w-[180px]">
<SelectValue placeholder="Default: English" />
</SelectTrigger>
<SelectContent>
{ocrLlanguages.map((lang) => (
<SelectItem key={lang.code} value={lang.code}>{lang.name}</SelectItem>
))}
</SelectContent>
</Select>
</div>
<div className="flex items-center gap-2">
<Label htmlFor="llmProvider">LLM Provider</Label>
<Select id="llmProvider" value={llmProvider} onValueChange={setLlmProvider}>
<SelectTrigger className="w-[180px]">
<SelectValue placeholder="Default: Chat GPT" />
</SelectTrigger>
<SelectContent>
<SelectItem key="chatgpt" value="chatgpt">Default: Chat GPT</SelectItem>
<SelectItem key="ollama" value="ollama">Ollama</SelectItem>
</SelectContent>
</Select>
</div>
<div className="flex items-center gap-2">
<Label htmlFor="llmMode">LLM Mode</Label>
<Select id="llmMode" value={llmMode} onValueChange={setLlmMode}>
<SelectTrigger className="w-[180px]">
<SelectValue placeholder="Default: Cloud Parse + Chat" />
</SelectTrigger>
<SelectContent>
<SelectItem key="cloud_parse_cloud_chat" value="cloud_parse_cloud_chat">Cloud Parse + Chat</SelectItem>
<SelectItem key="local_parse_local_caht" value="local_parse_local_chat">Local Parse + Chat</SelectItem>
<SelectItem key="local_parse_cloud_chat" value="local_parse_cloud_chat">Local Parse + Cloud Chat</SelectItem>
</SelectContent>
</Select>
</div>

<div>
<Label htmlFor="ollamaUrl">Ollama URL:</Label>
<Input
type="text"
id="ollamaUrl"
{...register("ollamaUrl", { required: false, validate: {
ollamaUrlValidator: (value) => true
}} )}
/>
</div>
</div>
<div className="text-xs">
Local models are used for processing data on local server where it IS NOT STORED. You can use Local models to remove personal information from attachments before sending it to Cloud AI providers or used it for all purposes (Local Parse + Cloud Chat mode).
</div>

</div>
</CardContent>
<CardHeader>Cloud AI providers:</CardHeader>
<CardContent className="space-y-2">
<Label htmlFor="chatGptApiKey">ChatGPT API Key</Label>
<Input
Expand Down
7 changes: 4 additions & 3 deletions src/contexts/chat-context.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ export const ChatContextProvider: React.FC<PropsWithChildren> = ({ children }) =

const config = useContext(ConfigContext);
const checkApiConfig = async () => {
const apiKey = await config?.serverConfig['chatGptApiKey'] as string;
const apiKey = await config?.getServerConfig('chatGptApiKey') as string;
if (!apiKey) {
config?.setConfigDialogOpen(true);
toast.info('Please enter Chat GPT API Key first');
Expand All @@ -96,7 +96,7 @@ export const ChatContextProvider: React.FC<PropsWithChildren> = ({ children }) =
const aiProvider = async () => {
await checkApiConfig();
const aiProvider = createOpenAI({
apiKey: await config?.serverConfig['chatGptApiKey'] as string
apiKey: await config?.getServerConfig('chatGptApiKey') as string
})
return aiProvider.chat('gpt-4o') //gpt-4o-2024-05-13
}
Expand Down Expand Up @@ -157,8 +157,9 @@ export const ChatContextProvider: React.FC<PropsWithChildren> = ({ children }) =
newMessages.push(newlyCreatedOne);
}

// TODO: Add multi LLM support - messages hould be sent to different LLMs based on the message llm model - so the messages should be grouped in threads

// removing attachments from previously sent messages
// TODO: remove the workaround with "prev_sent_attachments" by extending the MessageEx type with our own to save space for it
aiApiCall([...messages.map(msg => {
return Object.assign(msg, { experimental_attachments: null, prev_sent_attachments: msg.experimental_attachments })
}), ...newMessages], envelope.onResult);
Expand Down
Loading