Home Explore Help
Register Sign In
marx
/
llama-recipes
1
0
Fork 0
Files Issues 0 Pull Requests 0 Wiki
Browse Source

aliginng the pad token with HF latest

Hamid Shojanazeri 1 year ago
parent
7ec390bfc8
commit
557e881fcc
1 changed files with 3 additions and 5 deletions
Unified View Show Diff Stats
  1. 3 5
      inference/chat_completion.py

+ 3 - 5
inference/chat_completion.py
View File 

@@ -62,13 +62,11 @@ def main(
 
     tokenizer = LlamaTokenizer.from_pretrained(model_name)
 
     tokenizer = LlamaTokenizer.from_pretrained(model_name)
 
     tokenizer.add_special_tokens(
 
     tokenizer.add_special_tokens(
 
         {
 
         {
 
-            "eos_token": "</s>",
 
-            "bos_token": "</s>",
 
-            "unk_token": "</s>",
 
-            "pad_token": "[PAD]",
 
+         
+            "pad_token": "<PAD>",
 
         }
 
         }
 
     )
 
     )
 
-
 
+    
     chats = format_tokens(dialogs, tokenizer)
 
     chats = format_tokens(dialogs, tokenizer)
 
 
 
     with torch.no_grad():
 
     with torch.no_grad():