dist_checkpoint_root_folder: str="PATH/to/save/FSDP/model"# will be used if using FSDP
dist_checkpoint_folder: str="fine-tuned"# will be used if using FSDP
save_optimizer: bool=False# will be used if using FSDP
- use_fast_kernels: bool = False,# Enable using SDPA from PyTroch Accelerated Transformers, make use Flash Attention and Xformer memory-efficient kernels
+ use_fast_kernels: bool = False# Enable using SDPA from PyTroch Accelerated Transformers, make use Flash Attention and Xformer memory-efficient kernels