Started inference engine

include/lm/training/data_loader.hpp

@@ -0,0 +1,36 @@
+// include/lm/training/data_loader.hpp
+#pragma once
+
+#include <vector>
+#include <string>
+#include <fstream>
+#include <random>
+#include "../core/tensor.hpp"
+#include "../tokenizer/bpe_tokenizer.hpp"
+
+namespace lm {
+
+class ConversationDataLoader {
+public:
+    ConversationDataLoader(const std::string& file_path, BPETokenizer& tokenizer, 
+                         size_t batch_size, size_t seq_length);
+    
+    bool has_next() const;
+    std::pair<Tensor, Tensor> next_batch(); // Returns (input, target) tensors
+    
+    void reset();
+    size_t num_batches() const;
+
+private:
+    BPETokenizer& tokenizer_;
+    size_t batch_size_;
+    size_t seq_length_;
+    std::vector<std::vector<int>> conversations_;
+    size_t current_index_;
+    
+    void load_conversations(const std::string& file_path);
+    std::vector<int> tokenize_conversation(const std::string& conversation);
+};
+
+} // namespace lm
+

include/lm/training/losses.hpp

@@ -0,0 +1,11 @@
+// include/lm/training/losses.hpp
+#pragma once
+
+#include "../core/tensor.hpp"
+
+namespace lm {
+
+Tensor cross_entropy_loss(const Tensor& logits, const Tensor& targets, const Tensor& mask = Tensor());
+
+} // namespace lm
+

include/lm/training/trainer (copy 1).hpp

@@ -0,0 +1,42 @@
+// include/lm/training/trainer.hpp
+#pragma once
+
+#include <string>
+#include "../models/language_model.hpp"
+#include "../optimizers/adam.hpp"
+
+namespace lm {
+namespace training {
+
+struct TrainingCheckpoint {
+    size_t epoch;
+    size_t iteration;
+    float loss;
+    
+    template <class Archive>
+    void serialize(Archive& archive) {
+        archive(epoch, iteration, loss);
+    }
+};
+
+class Trainer {
+private:
+    LanguageModel& model;
+    AdamOptimizer& optimizer;
+    
+public:
+    Trainer(LanguageModel& model, AdamOptimizer& optimizer);
+    
+    void train(const std::vector<std::string>& corpus, 
+               size_t num_epochs, 
+               size_t batch_size, 
+               size_t sequence_length);
+    
+    void save_checkpoint(const std::string& path, 
+                        const TrainingCheckpoint& checkpoint) const;
+    TrainingCheckpoint load_checkpoint(const std::string& path);
+};
+
+} // namespace training
+} // namespace lm
+