Started inference engine

include/lm/context_manager.hpp

@@ -0,0 +1,44 @@
+// context_manager.hpp
+#pragma once
+
+#include <vector>
+#include <string>
+#include <deque>
+#include "token_types.hpp"
+
+namespace lm {
+
+class ContextManager {
+public:
+    ContextManager(size_t max_context_tokens = 2048, 
+                  size_t max_turns = 20);
+    
+    void add_user_message(const std::string& message);
+    void add_assistant_message(const std::string& message);
+    void add_system_message(const std::string& message);
+    
+    std::string get_context() const;
+    std::vector<TokenID> get_context_tokens() const;
+    
+    void clear();
+    void prune_old_messages();
+    
+    size_t get_token_count() const { return current_token_count; }
+    size_t get_turn_count() const { return conversation_turns.size(); }
+
+private:
+    struct ConversationTurn {
+        std::string role;  // "user", "assistant", or "system"
+        std::string content;
+        size_t token_count;
+    };
+    
+    std::deque<ConversationTurn> conversation_turns;
+    size_t max_context_tokens;
+    size_t max_turns;
+    size_t current_token_count;
+    
+    void add_message(const std::string& role, const std::string& content);
+};
+
+} // namespace lm

include/lm/conversation.hpp

@@ -0,0 +1,187 @@
+// include/lm/conversation.hpp
+#pragma once
+
+#include <string>
+#include <vector>
+#include <map>
+#include <chrono>
+#include <memory>
+#include <cereal/types/vector.hpp>
+#include <cereal/types/map.hpp>
+#include <cereal/types/string.hpp>
+#include <cereal/types/chrono.hpp>
+#include <cereal/types/memory.hpp>
+#include <cereal/archives/binary.hpp>
+#include <cereal/types/utility.hpp> // For std::pair serialization
+
+namespace lm {
+
+// Enum for different speaker types
+enum class SpeakerType {
+    USER,
+    ASSISTANT,
+    SYSTEM,
+    UNKNOWN
+};
+
+// Convert SpeakerType to string
+inline std::string speaker_type_to_string(SpeakerType type) {
+    switch (type) {
+        case SpeakerType::USER: return "user";
+        case SpeakerType::ASSISTANT: return "assistant";
+        case SpeakerType::SYSTEM: return "system";
+        default: return "unknown";
+    }
+}
+
+// Convert string to SpeakerType
+inline SpeakerType string_to_speaker_type(const std::string& str) {
+    if (str == "user") return SpeakerType::USER;
+    if (str == "assistant") return SpeakerType::ASSISTANT;
+    if (str == "system") return SpeakerType::SYSTEM;
+    return SpeakerType::UNKNOWN;
+}
+
+// Represents a single turn in a conversation
+struct ConversationTurn {
+    SpeakerType speaker;
+    std::string text;
+    std::vector<int> tokens;  // Tokenized representation
+    std::chrono::system_clock::time_point timestamp;
+    std::map<std::string, std::string> metadata;  // Additional metadata
+    
+    ConversationTurn(SpeakerType speaker_type = SpeakerType::UNKNOWN, 
+                    const std::string& text = "",
+                    const std::map<std::string, std::string>& metadata = {})
+        : speaker(speaker_type), text(text), metadata(metadata) {
+        timestamp = std::chrono::system_clock::now();
+    }
+    
+    // Cereal serialization
+    template <class Archive>
+    void serialize(Archive& archive) {
+        archive(
+            cereal::make_nvp("speaker", reinterpret_cast<int&>(speaker)),
+            cereal::make_nvp("text", text),
+            cereal::make_nvp("tokens", tokens),
+            cereal::make_nvp("timestamp", timestamp),
+            cereal::make_nvp("metadata", metadata)
+        );
+    }
+};
+
+// Represents a complete conversation with multiple turns
+struct Conversation {
+    std::vector<ConversationTurn> turns;
+    std::string domain;  // e.g., "customer_service", "general_chat", "technical_support"
+    std::string language;
+    std::map<std::string, std::string> metadata;
+    std::chrono::system_clock::time_point start_time;
+    std::chrono::system_clock::time_point end_time;
+    
+    Conversation(const std::string& domain = "general_chat",
+                const std::string& language = "en",
+                const std::map<std::string, std::string>& metadata = {})
+        : domain(domain), language(language), metadata(metadata) {
+        start_time = std::chrono::system_clock::now();
+    }
+    
+    // Add a turn to the conversation
+    void add_turn(SpeakerType speaker, const std::string& text,
+                 const std::map<std::string, std::string>& metadata = {}) {
+        turns.emplace_back(speaker, text, metadata);
+        end_time = std::chrono::system_clock::now();
+    }
+    
+    // Get the last turn
+    ConversationTurn& last_turn() {
+        if (turns.empty()) {
+            throw std::out_of_range("No turns in conversation");
+        }
+        return turns.back();
+    }
+    
+    // Get the number of turns
+    size_t size() const {
+        return turns.size();
+    }
+    
+    // Check if conversation is empty
+    bool empty() const {
+        return turns.empty();
+    }
+    
+    // Clear all turns
+    void clear() {
+        turns.clear();
+        start_time = std::chrono::system_clock::now();
+    }
+    
+    // Get conversation duration in seconds
+    double duration() const {
+        if (turns.empty()) return 0.0;
+        auto duration = end_time - start_time;
+        return std::chrono::duration<double>(duration).count();
+    }
+    
+    // Cereal serialization
+    template <class Archive>
+    void serialize(Archive& archive) {
+        archive(
+            cereal::make_nvp("turns", turns),
+            cereal::make_nvp("domain", domain),
+            cereal::make_nvp("language", language),
+            cereal::make_nvp("metadata", metadata),
+            cereal::make_nvp("start_time", start_time),
+            cereal::make_nvp("end_time", end_time)
+        );
+    }
+};
+
+// Helper functions for conversation processing
+namespace conversation_utils {
+
+// Extract text from a range of turns
+inline std::string extract_text(const std::vector<ConversationTurn>& turns,
+                               size_t start_idx = 0, size_t end_idx = 0) {
+    if (end_idx == 0) end_idx = turns.size();
+    if (start_idx >= end_idx || end_idx > turns.size()) return "";
+    
+    std::string result;
+    for (size_t i = start_idx; i < end_idx; i++) {
+        result += speaker_type_to_string(turns[i].speaker) + ": " + turns[i].text + "\n";
+    }
+    return result;
+}
+
+// Create a training pair from conversation turns
+inline std::pair<std::string, std::string> create_training_pair(
+    const std::vector<ConversationTurn>& turns, size_t context_length) {
+    
+    if (turns.size() < 2) return {"", ""};
+    
+    // Use the last 'context_length' turns as context (excluding the last turn)
+    size_t start_idx = turns.size() > context_length + 1 ? 
+                      turns.size() - context_length - 1 : 0;
+    size_t end_idx = turns.size() - 1;
+    
+    std::string context = extract_text(turns, start_idx, end_idx);
+    std::string target = turns.back().text;
+    
+    return {context, target};
+}
+
+// Calculate turns-based context window
+inline std::vector<ConversationTurn> get_context_window(
+    const std::vector<ConversationTurn>& turns, size_t max_turns) {
+    
+    if (turns.size() <= max_turns) return turns;
+    
+    return std::vector<ConversationTurn>(
+        turns.end() - max_turns, turns.end());
+}
+
+} // namespace conversation_utils
+
+} // namespace lm
+

include/lm/conversation_manager.hpp

@@ -0,0 +1,72 @@
+// include/lm/conversation_manager.hpp
+#pragma once
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <unordered_map>
+#include <mutex>
+#include "conversation.hpp"
+
+namespace lm {
+
+class ConversationManager {
+public:
+    ConversationManager();
+    ~ConversationManager();
+    
+    // Create a new conversation
+    std::string create_conversation(const std::string& title = "");
+    
+    // Get a conversation by ID
+    std::shared_ptr<Conversation> get_conversation(const std::string& id);
+    
+    // Get all conversation IDs
+    std::vector<std::string> list_conversations() const;
+    
+    // Add a message to a conversation
+    void add_message(const std::string& conversation_id, 
+                     const std::string& role, 
+                     const std::string& content);
+    
+    // Get conversation history
+    std::vector<ConversationTurn> get_history(const std::string& conversation_id) const;
+    
+    // Save conversations to disk
+    bool save_conversations(const std::string& path) const;
+    
+    // Load conversations from disk
+    bool load_conversations(const std::string& path);
+    
+    // Delete a conversation
+    bool delete_conversation(const std::string& id);
+    
+    // Set conversation title
+    void set_title(const std::string& conversation_id, const std::string& title);
+    
+    // Get conversation title
+    std::string get_title(const std::string& conversation_id) const;
+    
+    // Get conversation metadata
+    std::map<std::string, std::string> get_metadata(const std::string& conversation_id) const;
+    
+    // Update conversation metadata
+    void update_metadata(const std::string& conversation_id, 
+                         const std::map<std::string, std::string>& metadata);
+    
+    // Clear all conversations
+    void clear();
+    
+    // Get number of conversations
+    size_t count() const;
+
+private:
+    std::unordered_map<std::string, std::shared_ptr<Conversation>> conversations_;
+    mutable std::mutex mutex_;
+    
+    // Generate a unique ID for conversations
+    std::string generate_id() const;
+};
+
+} // namespace lm
+

include/lm/conversation_serializer.hpp

@@ -0,0 +1,36 @@
+// include/lm/conversation_serialization.hpp
+#pragma once
+
+#include "conversation.hpp"
+#include <cereal/types/vector.hpp>
+#include <cereal/types/map.hpp>
+#include <cereal/types/string.hpp>
+#include <cereal/types/chrono.hpp>
+
+namespace lm {
+
+template <class Archive>
+void serialize(Archive& archive, ConversationTurn& turn) {
+    archive(
+        cereal::make_nvp("speaker", static_cast<int&>(turn.speaker)),
+        cereal::make_nvp("text", turn.text),
+        cereal::make_nvp("tokens", turn.tokens),
+        cereal::make_nvp("timestamp", turn.timestamp),
+        cereal::make_nvp("metadata", turn.metadata)
+    );
+}
+
+template <class Archive>
+void serialize(Archive& archive, Conversation& conv) {
+    archive(
+        cereal::make_nvp("turns", conv.turns),
+        cereal::make_nvp("domain", conv.domain),
+        cereal::make_nvp("language", conv.language),
+        cereal::make_nvp("metadata", conv.metadata),
+        cereal::make_nvp("start_time", conv.start_time),
+        cereal::make_nvp("end_time", conv.end_time)
+    );
+}
+
+} // namespace lm
+

include/lm/core/tensor_pool.hpp

@@ -0,0 +1,82 @@
+#pragma once
+
+#include "tensor.hpp"
+#include <vector>
+#include <memory>
+#include <unordered_map>
+#include <mutex>
+#include <stdexcept>
+
+namespace lm {
+
+class TensorPool {
+private:
+    struct TensorKey {
+        std::vector<size_t> shape;
+        bool requires_grad;
+
+        bool operator==(const TensorKey& other) const {
+            return shape == other.shape && requires_grad == other.requires_grad;
+        }
+    };
+
+    struct KeyHash {
+        std::size_t operator()(const TensorKey& k) const {
+            std::size_t seed = k.shape.size();
+            for (auto& i : k.shape) {
+                seed ^= i + 0x9e3779b9 + (seed << 6) + (seed >> 2);
+            }
+            seed ^= k.requires_grad + 0x9e3779b9 + (seed << 6) + (seed >> 2);
+            return seed;
+        }
+    };
+
+    std::unordered_map<TensorKey, std::vector<std::unique_ptr<Tensor>>, KeyHash> pool_;
+    mutable std::mutex mutex_;  // Make mutex mutable
+
+public:
+    TensorPool() = default;
+
+    std::unique_ptr<Tensor> acquire(const std::vector<size_t>& shape, bool requires_grad = false) {
+        TensorKey key{shape, requires_grad};
+        std::lock_guard<std::mutex> lock(mutex_);
+
+        auto it = pool_.find(key);
+        if (it != pool_.end() && !it->second.empty()) {
+            auto tensor = std::move(it->second.back());
+            it->second.pop_back();
+            return tensor;
+        }
+
+        return std::make_unique<Tensor>(shape, requires_grad);
+    }
+
+    void release(std::unique_ptr<Tensor> tensor) {
+        if (!tensor) return;
+
+        TensorKey key{tensor->shape(), tensor->requires_grad()};
+        std::lock_guard<std::mutex> lock(mutex_);
+
+        // Reset tensor state before pooling
+        tensor->zero_grad();
+        tensor->data().setZero();
+        
+        pool_[key].push_back(std::move(tensor));
+    }
+
+    void clear() {
+        std::lock_guard<std::mutex> lock(mutex_);
+        pool_.clear();
+    }
+
+    size_t size() const {
+        std::lock_guard<std::mutex> lock(mutex_);
+        size_t total = 0;
+        for (const auto& entry : pool_) {
+            total += entry.second.size();
+        }
+        return total;
+    }
+};
+
+} // namespace lm

include/lm/generation/sampler.hpp

@@ -0,0 +1,54 @@
+#pragma once
+
+#include "../core/tensor.hpp"
+#include <vector>
+#include <random>
+#include <algorithm>
+#include <numeric>
+
+namespace lm {
+
+class Sampler {
+public:
+    virtual ~Sampler() = default;
+    virtual int sample(const Tensor& logits) = 0;
+};
+
+class GreedySampler : public Sampler {
+public:
+    int sample(const Tensor& logits) override;
+};
+
+class RandomSampler : public Sampler {
+public:
+    RandomSampler(float temperature = 1.0);
+    int sample(const Tensor& logits) override;
+    
+private:
+    float temperature_;
+    std::mt19937 gen_;
+};
+
+class TopKSampler : public Sampler {
+public:
+    TopKSampler(int k, float temperature = 1.0);
+    int sample(const Tensor& logits) override;
+    
+private:
+    int k_;
+    float temperature_;
+    std::mt19937 gen_;
+};
+
+class TopPSampler : public Sampler {
+public:
+    TopPSampler(float p, float temperature = 1.0);
+    int sample(const Tensor& logits) override;
+    
+private:
+    float p_;
+    float temperature_;
+    std::mt19937 gen_;
+};
+
+} // namespace lm

include/lm/models/attention (copy 1).hpp

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "lm/core/tensor.hpp"
+#include <vector>
+#include <memory>
+
+namespace lm {
+
+class MultiHeadAttention {
+public:
+    MultiHeadAttention(size_t d_model, size_t num_heads, float dropout = 0.1f);
+    
+    std::vector<Tensor> parameters() const;
+    void set_training(bool training);
+    Tensor forward(const Tensor& query, const Tensor& key, const Tensor& value, 
+                   const Tensor& mask = Tensor()) const;
+    
+private:
+    Tensor split_heads(const Tensor& x) const;
+    Tensor combine_heads(const Tensor& x) const;
+    Tensor scaled_dot_product_attention(const Tensor& q, const Tensor& k, 
+                                        const Tensor& v, const Tensor& mask) const;
+    Tensor apply_dropout(const Tensor& input, float dropout_rate) const;
+    
+    size_t d_model_;
+    size_t num_heads_;
+    size_t d_k_;
+    float dropout_;
+    bool training_ = false;
+    
+    Tensor w_q_;
+    Tensor w_k_;
+    Tensor w_v_;
+    Tensor w_o_;
+};
+
+} // namespace lm

include/lm/models/conversation_model.hpp

@@ -0,0 +1,54 @@
+// Enhanced conversation_model.hpp
+#pragma once
+
+#include "transformer_model.hpp"
+#include "bpe_tokenizer.hpp"
+#include "context_manager.hpp"
+#include <string>
+#include <vector>
+#include <memory>
+
+namespace lm {
+
+class ConversationModel {
+public:
+    ConversationModel(size_t vocab_size, 
+                     size_t d_model = 512, 
+                     size_t n_layers = 6, 
+                     size_t n_heads = 8,
+                     size_t d_ff = 2048,
+                     float dropout = 0.1);
+    
+    // Train the model
+    void train(const std::vector<std::string>& conversations);
+    
+    // Generate a response with context management
+    std::string generate_response(const std::string& user_input);
+    
+    // Context management
+    void clear_context();
+    void set_system_prompt(const std::string& prompt);
+    size_t get_context_token_count() const;
+    
+    // Save and load
+    bool save_model(const std::string& path);
+    bool load_model(const std::string& path);
+    
+    // Set tokenizer
+    void set_tokenizer(std::shared_ptr<BPETokenizer> tokenizer) { 
+        tokenizer_ = tokenizer; 
+        context_manager_ = std::make_unique<ContextManager>(2048, 20);
+    }
+
+private:
+    std::shared_ptr<BPETokenizer> tokenizer_;
+    std::unique_ptr<TransformerModel> transformer_;
+    std::unique_ptr<ContextManager> context_manager_;
+    std::string system_prompt_;
+    
+    // Format conversation for training
+    std::string format_conversation(const std::vector<std::string>& turns);
+};
+
+} // namespace lm
+

include/lm/models/feed_forward (copy 1).hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "lm/core/tensor.hpp"
+#include <vector>
+
+namespace lm {
+
+class FeedForward {
+public:
+    FeedForward(size_t d_model, size_t d_ff, float dropout = 0.1f);
+    
+    std::vector<Tensor> parameters() const;
+    void set_training(bool training);
+    Tensor forward(const Tensor& input) const;
+    
+private:
+    Tensor apply_dropout(const Tensor& input, float dropout_rate) const;
+    Tensor gelu(const Tensor& input) const;
+    
+    size_t d_model_;
+    size_t d_ff_;
+    float dropout_;
+    bool training_ = false;
+    
+    Tensor w1_;
+    Tensor b1_;
+    Tensor w2_;
+    Tensor b2_;
+};
+
+} // namespace lm
+

include/lm/models/language_model (copy 1).hpp

@@ -0,0 +1,34 @@
+// include/lm/models/language_model.hpp
+#pragma once
+
+#include <vector>
+#include <cstdint>
+#include <string>
+#include "../core/tensor.hpp"
+
+namespace lm {
+
+using TokenID = uint32_t;
+
+class LanguageModel {
+public:
+    virtual ~LanguageModel() = default;
+    
+    // Pure virtual methods that must be implemented
+    virtual std::vector<Tensor> get_parameters() const = 0;
+    virtual void set_parameters(const std::vector<Tensor>& params) = 0;
+    virtual Tensor forward(const std::vector<TokenID>& input) = 0;
+    virtual Tensor forward(const std::vector<TokenID>& input, 
+                          const std::vector<TokenID>& targets) = 0;
+    
+    // Optional virtual methods with default implementations
+    virtual size_t get_vocab_size() const { return 0; }
+    virtual size_t get_max_sequence_length() const { return 0; }
+    
+    // Serialization
+    virtual void save(const std::string& path) const = 0;
+    virtual void load(const std::string& path) = 0;
+};
+
+} // namespace lm
+

include/lm/models/transformer_block (copy 1).hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "lm/core/tensor.hpp"
+#include "lm/models/attention.hpp"
+#include "lm/models/feed_forward.hpp"
+#include "lm/models/layer_norm.hpp"
+#include <memory>
+#include <vector>
+
+namespace lm {
+
+class TransformerBlock {
+public:
+    TransformerBlock(size_t d_model, size_t num_heads, size_t d_ff, float dropout);
+    
+    std::vector<Tensor> parameters() const;
+    void set_training(bool training);
+    Tensor forward(const Tensor& input, const Tensor& mask = Tensor()) const;
+    
+private:
+    size_t d_model_, num_heads_, d_ff_;
+    float dropout_;
+    bool training_ = false;
+    
+    std::unique_ptr<MultiHeadAttention> attention_;
+    std::unique_ptr<FeedForward> feed_forward_;
+    std::unique_ptr<LayerNorm> norm1_;
+    std::unique_ptr<LayerNorm> norm2_;
+};
+
+} // namespace lm
+

include/lm/models/transformer_model.hpp

@@ -0,0 +1,60 @@
+// transformer_model.hpp
+#pragma once
+
+#include <vector>
+#include <cstdint>
+#include <memory>
+#include <cmath>
+#include <random>
+#include <iostream>
+#include "lm/tokenizer/token_types.hpp"
+
+namespace lm {
+
+class TransformerModel {
+public:
+    TransformerModel(size_t vocab_size, 
+                    size_t d_model = 512, 
+                    size_t n_layers = 6, 
+                    size_t n_heads = 8,
+                    size_t d_ff = 2048,
+                    float dropout = 0.1);
+    
+    ~TransformerModel();
+    
+    // Forward pass
+    std::vector<float> forward(const std::vector<TokenID>& input_tokens);
+    
+    // Training methods
+    void train_step(const std::vector<TokenID>& input_tokens, 
+                   const std::vector<TokenID>& target_tokens);
+    float calculate_loss(const std::vector<float>& logits, 
+                        const std::vector<TokenID>& targets);
+    
+    // Generation methods
+    std::vector<TokenID> generate(const std::vector<TokenID>& context, 
+                                 size_t max_length = 100,
+                                 float temperature = 1.0);
+    
+    // Serialization
+    bool save(const std::string& filename);
+    bool load(const std::string& filename);
+    
+    // Get model info
+    size_t get_vocab_size() const { return vocab_size_; }
+    size_t get_d_model() const { return d_model_; }
+
+private:
+    class Impl;
+    std::unique_ptr<Impl> pimpl_;
+    
+    // Model parameters
+    size_t vocab_size_;
+    size_t d_model_;
+    size_t n_layers_;
+    size_t n_heads_;
+    size_t d_ff_;
+    float dropout_;
+};
+
+} // namespace lm

include/lm/optimizers/adam (copy 1).hpp

@@ -0,0 +1,80 @@
+// include/lm/optimizers/adam.hpp
+#pragma once
+
+#include <vector>
+#include <cmath>
+#include <cereal/types/vector.hpp>
+#include <cereal/archives/binary.hpp>
+#include "../core/tensor.hpp"
+
+namespace lm {
+
+class AdamOptimizer {
+private:
+    std::vector<Tensor> m;  // First moment vector
+    std::vector<Tensor> v;  // Second moment vector
+    size_t t;               // Timestep
+    float beta1;
+    float beta2;
+    float epsilon;
+    float learning_rate;
+
+public:
+    AdamOptimizer(float lr = 0.001, float b1 = 0.9, float b2 = 0.999, float eps = 1e-8);
+    
+    void update(std::vector<Tensor>& parameters, 
+                const std::vector<Tensor>& gradients);
+    
+    // Initialize moment vectors for parameters
+    void initialize_moments(const std::vector<Tensor>& parameters);
+    
+    // Reset the optimizer state
+    void reset();
+    
+    // Step function for compatibility with existing code
+    void step(std::vector<Tensor>& parameters) {
+        std::vector<Tensor> gradients;
+        for (auto& param : parameters) {
+            if (param.requires_grad()) {
+                gradients.push_back(param.grad());
+            } else {
+                gradients.push_back(Tensor::zeros(param.shape(), false));
+            }
+        }
+        update(parameters, gradients);
+    }
+    
+    void zero_grad(std::vector<Tensor>& parameters) {
+        for (auto& param : parameters) {
+            if (param.requires_grad()) {
+                param.zero_grad();
+            }
+        }
+    }
+    
+    // Serialization methods
+    void save_state(const std::string& path) const;
+    void load_state(const std::string& path);
+    
+    // Cereal serialization
+    template <class Archive>
+    void serialize(Archive& archive) {
+        archive(
+            cereal::make_nvp("m", m),
+            cereal::make_nvp("v", v),
+            cereal::make_nvp("t", t),
+            cereal::make_nvp("beta1", beta1),
+            cereal::make_nvp("beta2", beta2),
+            cereal::make_nvp("epsilon", epsilon),
+            cereal::make_nvp("learning_rate", learning_rate)
+        );
+    }
+    
+    // Getters for state inspection
+    size_t get_timestep() const { return t; }
+    float get_learning_rate() const { return learning_rate; }
+    void set_learning_rate(float lr) { learning_rate = lr; }
+};
+
+} // namespace lm
+

include/lm/runtime/init (copy 1).hpp

@@ -0,0 +1,54 @@
+// Runtime Initialization Header File
+
+//Here's the complete `include/lm/runtime/init.hpp` file:
+
+//```cpp
+#pragma once
+
+#include <string>
+#include <nlohmann/json.hpp>
+#include <filesystem>
+
+namespace lm::runtime {
+
+class SystemState {
+public:
+    // Singleton access
+    static SystemState& get_instance();
+    
+    // Initialize from JSON config
+    void initialize(const std::filesystem::path& config_path);
+    
+    // Configuration accessors
+    const nlohmann::json& config() const noexcept;
+    std::string get_string(const std::string& key) const;
+    int get_int(const std::string& key, int default_val = 0) const;
+    
+    // Subsystem states
+    bool is_tokenizer_ready() const noexcept;
+    bool is_model_loaded() const noexcept;
+
+private:
+    SystemState() = default; // Private constructor
+    nlohmann::json config_;
+    bool tokenizer_ready_ = false;
+    bool model_loaded_ = false;
+};
+
+} // namespace lm::runtime
+/*```
+
+This header provides the interface for the framework initialization system with:
+
+1. **Singleton pattern** for global system state access
+2. **JSON configuration** loading and access methods
+3. **Subsystem state tracking** for tokenizer and model
+4. **Type-safe configuration access** with default values
+
+The implementation (in the corresponding `.cpp` file) handles:
+- JSON configuration parsing and validation
+- Subsystem initialization sequencing
+- Error handling for malformed configurations
+- State management across the framework
+
+This initialization system provides a centralized way to configure and manage the LM framework components.*/

include/lm/runtime/shutdown (copy 1).hpp

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <nlohmann/json.hpp>
+#include <filesystem>
+#include <chrono>
+
+namespace lm::runtime {
+
+class ShutdownHandler {
+public:
+    // Serialize state to JSON
+    static void save_state(
+        const std::filesystem::path& output_path,
+        bool include_model_weights = false
+    );
+    
+    // Cleanup hooks
+    static void register_cleanup(void (*func)());
+    static void execute_cleanup();
+};
+
+} // namespace lm::runtime

include/lm/tokenizer/bpe_tokenizer (copy 1).hpp

@@ -0,0 +1,56 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <unordered_map>
+#include "token_types.hpp"
+
+namespace lm {
+
+class BPETokenizer {
+public:
+    BPETokenizer();
+    ~BPETokenizer();
+
+    // Training methods
+    void train(const std::vector<std::string>& corpus, size_t vocab_size);
+    
+    // Encoding/decoding methods
+    std::vector<TokenID> encode(const std::string& text) const;
+    std::string decode(const std::vector<TokenID>& tokens) const;
+    
+    // Vocabulary methods
+    size_t vocab_size() const;
+    
+    // Serialization methods
+    bool save(const std::string& filename) const;
+    bool load(const std::string& filename);
+    
+    // Special token methods
+    TokenID eos_token_id() const;
+    void set_eos_token_id(TokenID id);
+    
+    TokenID pad_token_id() const;
+    void set_pad_token_id(TokenID id);
+    
+    TokenID unk_token_id() const;
+    void set_unk_token_id(TokenID id);
+
+    // Add special tokens to vocabulary
+    void add_special_token(const std::string& token, TokenID id);
+    
+    // UTF-8 validation method
+    //bool is_valid_utf8_asm(const char* str, size_t length);
+
+    // Debug methods
+    void enable_debug_logging(bool enable);
+    void dump_vocabulary() const;
+    void dump_merges() const;
+
+private:
+    class Impl;
+    std::unique_ptr<Impl> pimpl_;
+};
+
+} // namespace lm

include/lm/tokenizer/token_types.hpp

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <cstdint>
+
+namespace lm {
+
+using TokenID = uint32_t;
+
+} // namespace lm
+

include/lm/tokenizer/unicode_utils (copy 1).hpp

@@ -0,0 +1,42 @@
+//# Unicode Utilities Header File
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include <cstdint>
+
+namespace lm::unicode {
+
+// Unicode character representation
+struct CodePoint {
+    uint32_t value;
+    std::string utf8;  // UTF-8 representation
+};
+
+// Check if a code point is whitespace
+bool is_whitespace(uint32_t codepoint);
+
+// Check if a code point is punctuation
+bool is_punctuation(uint32_t codepoint);
+
+// Check if a code point is a control character
+bool is_control(uint32_t codepoint);
+
+// Normalize Unicode text (NFC normalization)
+std::string normalize(const std::string& text);
+
+// Split text into Unicode code points
+std::vector<CodePoint> to_code_points(const std::string& text);
+
+// Convert code points back to UTF-8 string
+std::string from_code_points(const std::vector<CodePoint>& code_points);
+
+// Unicode-aware string split (handles Unicode whitespace)
+std::vector<std::string> unicode_split(const std::string& text);
+
+// Unicode-aware character boundaries
+std::vector<std::string> split_on_character_boundaries(const std::string& text);
+
+} // namespace lm::unicode
+

include/lm/training/data_loader.hpp

@@ -0,0 +1,36 @@
+// include/lm/training/data_loader.hpp
+#pragma once
+
+#include <vector>
+#include <string>
+#include <fstream>
+#include <random>
+#include "../core/tensor.hpp"
+#include "../tokenizer/bpe_tokenizer.hpp"
+
+namespace lm {
+
+class ConversationDataLoader {
+public:
+    ConversationDataLoader(const std::string& file_path, BPETokenizer& tokenizer, 
+                         size_t batch_size, size_t seq_length);
+    
+    bool has_next() const;
+    std::pair<Tensor, Tensor> next_batch(); // Returns (input, target) tensors
+    
+    void reset();
+    size_t num_batches() const;
+
+private:
+    BPETokenizer& tokenizer_;
+    size_t batch_size_;
+    size_t seq_length_;
+    std::vector<std::vector<int>> conversations_;
+    size_t current_index_;
+    
+    void load_conversations(const std::string& file_path);
+    std::vector<int> tokenize_conversation(const std::string& conversation);
+};
+
+} // namespace lm
+

include/lm/training/losses.hpp

@@ -0,0 +1,11 @@
+// include/lm/training/losses.hpp
+#pragma once
+
+#include "../core/tensor.hpp"
+
+namespace lm {
+
+Tensor cross_entropy_loss(const Tensor& logits, const Tensor& targets, const Tensor& mask = Tensor());
+
+} // namespace lm
+

include/lm/training/trainer (copy 1).hpp

@@ -0,0 +1,42 @@
+// include/lm/training/trainer.hpp
+#pragma once
+
+#include <string>
+#include "../models/language_model.hpp"
+#include "../optimizers/adam.hpp"
+
+namespace lm {
+namespace training {
+
+struct TrainingCheckpoint {
+    size_t epoch;
+    size_t iteration;
+    float loss;
+    
+    template <class Archive>
+    void serialize(Archive& archive) {
+        archive(epoch, iteration, loss);
+    }
+};
+
+class Trainer {
+private:
+    LanguageModel& model;
+    AdamOptimizer& optimizer;
+    
+public:
+    Trainer(LanguageModel& model, AdamOptimizer& optimizer);
+    
+    void train(const std::vector<std::string>& corpus, 
+               size_t num_epochs, 
+               size_t batch_size, 
+               size_t sequence_length);
+    
+    void save_checkpoint(const std::string& path, 
+                        const TrainingCheckpoint& checkpoint) const;
+    TrainingCheckpoint load_checkpoint(const std::string& path);
+};
+
+} // namespace training
+} // namespace lm
+