|
@@ -33,8 +33,8 @@ class train_config:
|
|
dist_checkpoint_root_folder: str="PATH/to/save/FSDP/model" # will be used if using FSDP
|
|
dist_checkpoint_root_folder: str="PATH/to/save/FSDP/model" # will be used if using FSDP
|
|
dist_checkpoint_folder: str="fine-tuned" # will be used if using FSDP
|
|
dist_checkpoint_folder: str="fine-tuned" # will be used if using FSDP
|
|
save_optimizer: bool=False # will be used if using FSDP
|
|
save_optimizer: bool=False # will be used if using FSDP
|
|
- use_fast_kernels: bool = False, # Enable using SDPA from PyTroch Accelerated Transformers, make use Flash Attention and Xformer memory-efficient kernels
|
|
|
|
|
|
+ use_fast_kernels: bool = False # Enable using SDPA from PyTroch Accelerated Transformers, make use Flash Attention and Xformer memory-efficient kernels
|
|
|
|
|
|
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
+
|