Initial commit

include/lm/runtime/init.hpp

@@ -0,0 +1,54 @@
+// Runtime Initialization Header File
+
+//Here's the complete `include/lm/runtime/init.hpp` file:
+
+//```cpp
+#pragma once
+
+#include <string>
+#include <nlohmann/json.hpp>
+#include <filesystem>
+
+namespace lm::runtime {
+
+class SystemState {
+public:
+    // Singleton access
+    static SystemState& get_instance();
+    
+    // Initialize from JSON config
+    void initialize(const std::filesystem::path& config_path);
+    
+    // Configuration accessors
+    const nlohmann::json& config() const noexcept;
+    std::string get_string(const std::string& key) const;
+    int get_int(const std::string& key, int default_val = 0) const;
+    
+    // Subsystem states
+    bool is_tokenizer_ready() const noexcept;
+    bool is_model_loaded() const noexcept;
+
+private:
+    SystemState() = default; // Private constructor
+    nlohmann::json config_;
+    bool tokenizer_ready_ = false;
+    bool model_loaded_ = false;
+};
+
+} // namespace lm::runtime
+/*```
+
+This header provides the interface for the framework initialization system with:
+
+1. **Singleton pattern** for global system state access
+2. **JSON configuration** loading and access methods
+3. **Subsystem state tracking** for tokenizer and model
+4. **Type-safe configuration access** with default values
+
+The implementation (in the corresponding `.cpp` file) handles:
+- JSON configuration parsing and validation
+- Subsystem initialization sequencing
+- Error handling for malformed configurations
+- State management across the framework
+
+This initialization system provides a centralized way to configure and manage the LM framework components.*/

include/lm/runtime/shutdown.hpp

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <nlohmann/json.hpp>
+#include <filesystem>
+#include <chrono>
+
+namespace lm::runtime {
+
+class ShutdownHandler {
+public:
+    // Serialize state to JSON
+    static void save_state(
+        const std::filesystem::path& output_path,
+        bool include_model_weights = false
+    );
+    
+    // Cleanup hooks
+    static void register_cleanup(void (*func)());
+    static void execute_cleanup();
+};
+
+} // namespace lm::runtime

include/lm/tokenizer/bpe_tokenizer.hpp

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include <map>
+#include <memory>
+#include <utility>
+#include <cstdint>  // For uint16_t
+#include <queue>
+#include <functional>
+
+namespace lm {
+    
+using TokenID = uint16_t;  // Support for 65k vocabulary
+
+class BPETokenizer {
+public:
+    BPETokenizer();
+    ~BPETokenizer();
+
+    // Training methods
+    void train(const std::vector<std::string>& corpus, size_t vocab_size = 30000);
+    void train_from_file(const std::string& filename, size_t vocab_size = 30000);
+
+    // Tokenization methods
+    std::vector<TokenID> encode(const std::string& text) const;
+    std::string decode(const std::vector<TokenID>& tokens) const;
+    
+    // Serialization
+    bool save(const std::string& filename) const;
+    bool load(const std::string& filename);
+
+    // Vocabulary access
+    size_t vocab_size() const;
+    std::string id_to_token(TokenID id) const;
+    TokenID token_to_id(const std::string& token) const;
+
+    // Configuration
+    void set_unknown_token(const std::string& token);
+    void add_special_token(const std::string& token);
+    
+    // Unicode-specific methods
+    void set_normalization(bool enabled);
+    void set_byte_fallback(bool enabled);
+
+private:
+    struct Impl;
+    std::unique_ptr<Impl> pimpl_;
+};
+
+} // namespace lm

include/lm/tokenizer/unicode_utils.hpp

@@ -0,0 +1,59 @@
+//# Unicode Utilities Header File
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include <cstdint>
+
+namespace lm::unicode {
+
+// Unicode character representation
+struct CodePoint {
+    uint32_t value;
+    std::string utf8;  // UTF-8 representation
+};
+
+// Check if a code point is whitespace
+bool is_whitespace(uint32_t codepoint);
+
+// Check if a code point is punctuation
+bool is_punctuation(uint32_t codepoint);
+
+// Check if a code point is a control character
+bool is_control(uint32_t codepoint);
+
+// Normalize Unicode text (NFC normalization)
+std::string normalize(const std::string& text);
+
+// Split text into Unicode code points
+std::vector<CodePoint> to_code_points(const std::string& text);
+
+// Convert code points back to UTF-8 string
+std::string from_code_points(const std::vector<CodePoint>& code_points);
+
+// Unicode-aware string split (handles Unicode whitespace)
+std::vector<std::string> unicode_split(const std::string& text);
+
+// Unicode-aware character boundaries
+std::vector<std::string> split_on_character_boundaries(const std::string& text);
+
+} // namespace lm::unicode
+/*```
+
+This header provides a comprehensive set of Unicode utilities for the BPE tokenizer, including:
+
+1. **Code point representation** with both numeric value and UTF-8 string
+2. **Character classification** functions (whitespace, punctuation, control)
+3. **Text normalization** using Unicode NFC form
+4. **UTF-8 encoding/decoding** utilities
+5. **Unicode-aware text splitting** for proper tokenization
+
+These utilities enable the tokenizer to properly handle:
+- Multi-byte UTF-8 sequences
+- Unicode whitespace characters
+- Text normalization for consistent processing
+- Proper character boundary detection
+- Support for various writing systems (Latin, CJK, Arabic, etc.)
+
+The implementation in the corresponding `.cpp` file uses the ICU library for robust Unicode handling.*/