Started inference engine

include/lm/tokenizer/bpe_tokenizer (copy 1).hpp

@@ -0,0 +1,56 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <unordered_map>
+#include "token_types.hpp"
+
+namespace lm {
+
+class BPETokenizer {
+public:
+    BPETokenizer();
+    ~BPETokenizer();
+
+    // Training methods
+    void train(const std::vector<std::string>& corpus, size_t vocab_size);
+    
+    // Encoding/decoding methods
+    std::vector<TokenID> encode(const std::string& text) const;
+    std::string decode(const std::vector<TokenID>& tokens) const;
+    
+    // Vocabulary methods
+    size_t vocab_size() const;
+    
+    // Serialization methods
+    bool save(const std::string& filename) const;
+    bool load(const std::string& filename);
+    
+    // Special token methods
+    TokenID eos_token_id() const;
+    void set_eos_token_id(TokenID id);
+    
+    TokenID pad_token_id() const;
+    void set_pad_token_id(TokenID id);
+    
+    TokenID unk_token_id() const;
+    void set_unk_token_id(TokenID id);
+
+    // Add special tokens to vocabulary
+    void add_special_token(const std::string& token, TokenID id);
+    
+    // UTF-8 validation method
+    //bool is_valid_utf8_asm(const char* str, size_t length);
+
+    // Debug methods
+    void enable_debug_logging(bool enable);
+    void dump_vocabulary() const;
+    void dump_merges() const;
+
+private:
+    class Impl;
+    std::unique_ptr<Impl> pimpl_;
+};
+
+} // namespace lm

include/lm/tokenizer/token_types.hpp

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <cstdint>
+
+namespace lm {
+
+using TokenID = uint32_t;
+
+} // namespace lm
+

include/lm/tokenizer/unicode_utils (copy 1).hpp

@@ -0,0 +1,42 @@
+//# Unicode Utilities Header File
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include <cstdint>
+
+namespace lm::unicode {
+
+// Unicode character representation
+struct CodePoint {
+    uint32_t value;
+    std::string utf8;  // UTF-8 representation
+};
+
+// Check if a code point is whitespace
+bool is_whitespace(uint32_t codepoint);
+
+// Check if a code point is punctuation
+bool is_punctuation(uint32_t codepoint);
+
+// Check if a code point is a control character
+bool is_control(uint32_t codepoint);
+
+// Normalize Unicode text (NFC normalization)
+std::string normalize(const std::string& text);
+
+// Split text into Unicode code points
+std::vector<CodePoint> to_code_points(const std::string& text);
+
+// Convert code points back to UTF-8 string
+std::string from_code_points(const std::vector<CodePoint>& code_points);
+
+// Unicode-aware string split (handles Unicode whitespace)
+std::vector<std::string> unicode_split(const std::string& text);
+
+// Unicode-aware character boundaries
+std::vector<std::string> split_on_character_boundaries(const std::string& text);
+
+} // namespace lm::unicode
+