Started inference engine

include/lm/models/attention (copy 1).hpp

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "lm/core/tensor.hpp"
+#include <vector>
+#include <memory>
+
+namespace lm {
+
+class MultiHeadAttention {
+public:
+    MultiHeadAttention(size_t d_model, size_t num_heads, float dropout = 0.1f);
+    
+    std::vector<Tensor> parameters() const;
+    void set_training(bool training);
+    Tensor forward(const Tensor& query, const Tensor& key, const Tensor& value, 
+                   const Tensor& mask = Tensor()) const;
+    
+private:
+    Tensor split_heads(const Tensor& x) const;
+    Tensor combine_heads(const Tensor& x) const;
+    Tensor scaled_dot_product_attention(const Tensor& q, const Tensor& k, 
+                                        const Tensor& v, const Tensor& mask) const;
+    Tensor apply_dropout(const Tensor& input, float dropout_rate) const;
+    
+    size_t d_model_;
+    size_t num_heads_;
+    size_t d_k_;
+    float dropout_;
+    bool training_ = false;
+    
+    Tensor w_q_;
+    Tensor w_k_;
+    Tensor w_v_;
+    Tensor w_o_;
+};
+
+} // namespace lm

include/lm/models/conversation_model.hpp

@@ -0,0 +1,54 @@
+// Enhanced conversation_model.hpp
+#pragma once
+
+#include "transformer_model.hpp"
+#include "bpe_tokenizer.hpp"
+#include "context_manager.hpp"
+#include <string>
+#include <vector>
+#include <memory>
+
+namespace lm {
+
+class ConversationModel {
+public:
+    ConversationModel(size_t vocab_size, 
+                     size_t d_model = 512, 
+                     size_t n_layers = 6, 
+                     size_t n_heads = 8,
+                     size_t d_ff = 2048,
+                     float dropout = 0.1);
+    
+    // Train the model
+    void train(const std::vector<std::string>& conversations);
+    
+    // Generate a response with context management
+    std::string generate_response(const std::string& user_input);
+    
+    // Context management
+    void clear_context();
+    void set_system_prompt(const std::string& prompt);
+    size_t get_context_token_count() const;
+    
+    // Save and load
+    bool save_model(const std::string& path);
+    bool load_model(const std::string& path);
+    
+    // Set tokenizer
+    void set_tokenizer(std::shared_ptr<BPETokenizer> tokenizer) { 
+        tokenizer_ = tokenizer; 
+        context_manager_ = std::make_unique<ContextManager>(2048, 20);
+    }
+
+private:
+    std::shared_ptr<BPETokenizer> tokenizer_;
+    std::unique_ptr<TransformerModel> transformer_;
+    std::unique_ptr<ContextManager> context_manager_;
+    std::string system_prompt_;
+    
+    // Format conversation for training
+    std::string format_conversation(const std::vector<std::string>& turns);
+};
+
+} // namespace lm
+

include/lm/models/feed_forward (copy 1).hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "lm/core/tensor.hpp"
+#include <vector>
+
+namespace lm {
+
+class FeedForward {
+public:
+    FeedForward(size_t d_model, size_t d_ff, float dropout = 0.1f);
+    
+    std::vector<Tensor> parameters() const;
+    void set_training(bool training);
+    Tensor forward(const Tensor& input) const;
+    
+private:
+    Tensor apply_dropout(const Tensor& input, float dropout_rate) const;
+    Tensor gelu(const Tensor& input) const;
+    
+    size_t d_model_;
+    size_t d_ff_;
+    float dropout_;
+    bool training_ = false;
+    
+    Tensor w1_;
+    Tensor b1_;
+    Tensor w2_;
+    Tensor b2_;
+};
+
+} // namespace lm
+

include/lm/models/language_model (copy 1).hpp

@@ -0,0 +1,34 @@
+// include/lm/models/language_model.hpp
+#pragma once
+
+#include <vector>
+#include <cstdint>
+#include <string>
+#include "../core/tensor.hpp"
+
+namespace lm {
+
+using TokenID = uint32_t;
+
+class LanguageModel {
+public:
+    virtual ~LanguageModel() = default;
+    
+    // Pure virtual methods that must be implemented
+    virtual std::vector<Tensor> get_parameters() const = 0;
+    virtual void set_parameters(const std::vector<Tensor>& params) = 0;
+    virtual Tensor forward(const std::vector<TokenID>& input) = 0;
+    virtual Tensor forward(const std::vector<TokenID>& input, 
+                          const std::vector<TokenID>& targets) = 0;
+    
+    // Optional virtual methods with default implementations
+    virtual size_t get_vocab_size() const { return 0; }
+    virtual size_t get_max_sequence_length() const { return 0; }
+    
+    // Serialization
+    virtual void save(const std::string& path) const = 0;
+    virtual void load(const std::string& path) = 0;
+};
+
+} // namespace lm
+

include/lm/models/transformer_block (copy 1).hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "lm/core/tensor.hpp"
+#include "lm/models/attention.hpp"
+#include "lm/models/feed_forward.hpp"
+#include "lm/models/layer_norm.hpp"
+#include <memory>
+#include <vector>
+
+namespace lm {
+
+class TransformerBlock {
+public:
+    TransformerBlock(size_t d_model, size_t num_heads, size_t d_ff, float dropout);
+    
+    std::vector<Tensor> parameters() const;
+    void set_training(bool training);
+    Tensor forward(const Tensor& input, const Tensor& mask = Tensor()) const;
+    
+private:
+    size_t d_model_, num_heads_, d_ff_;
+    float dropout_;
+    bool training_ = false;
+    
+    std::unique_ptr<MultiHeadAttention> attention_;
+    std::unique_ptr<FeedForward> feed_forward_;
+    std::unique_ptr<LayerNorm> norm1_;
+    std::unique_ptr<LayerNorm> norm2_;
+};
+
+} // namespace lm
+

include/lm/models/transformer_model.hpp

@@ -0,0 +1,60 @@
+// transformer_model.hpp
+#pragma once
+
+#include <vector>
+#include <cstdint>
+#include <memory>
+#include <cmath>
+#include <random>
+#include <iostream>
+#include "lm/tokenizer/token_types.hpp"
+
+namespace lm {
+
+class TransformerModel {
+public:
+    TransformerModel(size_t vocab_size, 
+                    size_t d_model = 512, 
+                    size_t n_layers = 6, 
+                    size_t n_heads = 8,
+                    size_t d_ff = 2048,
+                    float dropout = 0.1);
+    
+    ~TransformerModel();
+    
+    // Forward pass
+    std::vector<float> forward(const std::vector<TokenID>& input_tokens);
+    
+    // Training methods
+    void train_step(const std::vector<TokenID>& input_tokens, 
+                   const std::vector<TokenID>& target_tokens);
+    float calculate_loss(const std::vector<float>& logits, 
+                        const std::vector<TokenID>& targets);
+    
+    // Generation methods
+    std::vector<TokenID> generate(const std::vector<TokenID>& context, 
+                                 size_t max_length = 100,
+                                 float temperature = 1.0);
+    
+    // Serialization
+    bool save(const std::string& filename);
+    bool load(const std::string& filename);
+    
+    // Get model info
+    size_t get_vocab_size() const { return vocab_size_; }
+    size_t get_d_model() const { return d_model_; }
+
+private:
+    class Impl;
+    std::unique_ptr<Impl> pimpl_;
+    
+    // Model parameters
+    size_t vocab_size_;
+    size_t d_model_;
+    size_t n_layers_;
+    size_t n_heads_;
+    size_t d_ff_;
+    float dropout_;
+};
+
+} // namespace lm