mild cleanup

don't need the space-prefixing here rn since main.cpp already does it
ggml-org · Mar 18, 2023 · 0acb5f5 · 0acb5f5
1 parent 8d32441
commit 0acb5f5
Showing 1 changed file with 3 additions and 12 deletions.
diff --git a/utils.cpp b/utils.cpp
@@ -303,7 +303,7 @@ struct llama_sp_symbol {
 
 struct llama_sp_bigram {
     struct comparator {
-        const bool operator()(llama_sp_bigram & l, llama_sp_bigram & r) {
+        bool operator()(llama_sp_bigram & l, llama_sp_bigram & r) {
             return (l.score < r.score) || (l.score == r.score && l.left > r.left);
         }
     };
@@ -409,7 +409,7 @@ struct llama_tokenizer {
         work_queue_.push(bigram);
     }
 
-    gpt_vocab const& vocab_;
+    const gpt_vocab & vocab_;
     std::vector<llama_sp_symbol> symbols_;
     llama_sp_bigram::queue work_queue_;
 };
@@ -426,16 +426,7 @@ std::vector<gpt_vocab::id> llama_tokenize(const gpt_vocab & vocab, std::string_v
         output.push_back(1);
     }
 
-    if (text[0] != ' ') {
-        std::string copy;
-
-        copy += ' ';
-        copy += text;
-        tokenizer.tokenize(copy, output);
-    } else {
-        tokenizer.tokenize(text, output);
-    }
-
+    tokenizer.tokenize(text, output);
     return output;
 }