Initial commit

include/lm/tokenizer/bpe_tokenizer.hpp

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include <map>
+#include <memory>
+#include <utility>
+#include <cstdint>  // For uint16_t
+#include <queue>
+#include <functional>
+
+namespace lm {
+    
+using TokenID = uint16_t;  // Support for 65k vocabulary
+
+class BPETokenizer {
+public:
+    BPETokenizer();
+    ~BPETokenizer();
+
+    // Training methods
+    void train(const std::vector<std::string>& corpus, size_t vocab_size = 30000);
+    void train_from_file(const std::string& filename, size_t vocab_size = 30000);
+
+    // Tokenization methods
+    std::vector<TokenID> encode(const std::string& text) const;
+    std::string decode(const std::vector<TokenID>& tokens) const;
+    
+    // Serialization
+    bool save(const std::string& filename) const;
+    bool load(const std::string& filename);
+
+    // Vocabulary access
+    size_t vocab_size() const;
+    std::string id_to_token(TokenID id) const;
+    TokenID token_to_id(const std::string& token) const;
+
+    // Configuration
+    void set_unknown_token(const std::string& token);
+    void add_special_token(const std::string& token);
+    
+    // Unicode-specific methods
+    void set_normalization(bool enabled);
+    void set_byte_fallback(bool enabled);
+
+private:
+    struct Impl;
+    std::unique_ptr<Impl> pimpl_;
+};
+
+} // namespace lm

include/lm/tokenizer/unicode_utils.hpp

@@ -0,0 +1,59 @@
+//# Unicode Utilities Header File
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include <cstdint>
+
+namespace lm::unicode {
+
+// Unicode character representation
+struct CodePoint {
+    uint32_t value;
+    std::string utf8;  // UTF-8 representation
+};
+
+// Check if a code point is whitespace
+bool is_whitespace(uint32_t codepoint);
+
+// Check if a code point is punctuation
+bool is_punctuation(uint32_t codepoint);
+
+// Check if a code point is a control character
+bool is_control(uint32_t codepoint);
+
+// Normalize Unicode text (NFC normalization)
+std::string normalize(const std::string& text);
+
+// Split text into Unicode code points
+std::vector<CodePoint> to_code_points(const std::string& text);
+
+// Convert code points back to UTF-8 string
+std::string from_code_points(const std::vector<CodePoint>& code_points);
+
+// Unicode-aware string split (handles Unicode whitespace)
+std::vector<std::string> unicode_split(const std::string& text);
+
+// Unicode-aware character boundaries
+std::vector<std::string> split_on_character_boundaries(const std::string& text);
+
+} // namespace lm::unicode
+/*```
+
+This header provides a comprehensive set of Unicode utilities for the BPE tokenizer, including:
+
+1. **Code point representation** with both numeric value and UTF-8 string
+2. **Character classification** functions (whitespace, punctuation, control)
+3. **Text normalization** using Unicode NFC form
+4. **UTF-8 encoding/decoding** utilities
+5. **Unicode-aware text splitting** for proper tokenization
+
+These utilities enable the tokenizer to properly handle:
+- Multi-byte UTF-8 sequences
+- Unicode whitespace characters
+- Text normalization for consistent processing
+- Proper character boundary detection
+- Support for various writing systems (Latin, CJK, Arabic, etc.)
+
+The implementation in the corresponding `.cpp` file uses the ICU library for robust Unicode handling.*/