|
@@ -23,8 +23,6 @@ from llama_recipes.policies import fpSixteen,bfSixteen_mixed, get_llama_wrapper
|
|
|
from llama_recipes.utils.memory_utils import MemoryTrace
|
|
|
from accelerate.utils import is_xpu_available, is_ccl_available
|
|
|
|
|
|
-from accelerate.utils import is_xpu_available, is_ccl_available
|
|
|
-
|
|
|
def set_tokenizer_params(tokenizer: LlamaTokenizer):
|
|
|
tokenizer.pad_token_id = 0
|
|
|
tokenizer.padding_side = "left"
|