Latest commit to Fred's git server

tests/tokenizer/test_openmtadapter.cpp

@@ -0,0 +1,12 @@
+#include "tokenizer/opennmt_adapter.hpp"
+#include <gtest/gtest.h>
+
+TEST(OpenNMTAdapter, ModeSwitch) {
+    OpenNMTAdapter tokenizer("config.json");
+    
+    auto bpe_tokens = tokenizer.encode("hello", OpenNMTAdapter::Mode::BPE);
+    auto sp_tokens = tokenizer.encode("hello", OpenNMTAdapter::Mode::SENTENCEPIECE);
+    
+    EXPECT_NE(bpe_tokens, sp_tokens); // Different tokenization schemes
+}
+