|
@@ -32,7 +32,7 @@
|
|
|
"outputs": [],
|
|
|
"source": [
|
|
|
"# %%bash\n",
|
|
|
- "# pip install transformers datasets accelerate sentencepiece protobuf==3.20 py7zr scipy peft bitsandbytes fire torch_tb_profiler ipywidgets\n",
|
|
|
+ "# pip install llama-recipes transformers datasets accelerate sentencepiece protobuf==3.20 py7zr scipy peft bitsandbytes fire torch_tb_profiler ipywidgets\n",
|
|
|
"# TRANSFORM=`python -c \"import transformers;print('/'.join(transformers.__file__.split('/')[:-1])+'/models/llama/convert_llama_weights_to_hf.py')\"`\n",
|
|
|
"# python ${TRANSFORM} --input_dir models --model_size 7B --output_dir models_hf/7B"
|
|
|
]
|
|
@@ -130,11 +130,8 @@
|
|
|
}
|
|
|
],
|
|
|
"source": [
|
|
|
- "from pathlib import Path\n",
|
|
|
- "import os\n",
|
|
|
- "import sys\n",
|
|
|
- "from utils.dataset_utils import get_preprocessed_dataset\n",
|
|
|
- "from configs.datasets import samsum_dataset\n",
|
|
|
+ "from llama_recipes.utils.dataset_utils import get_preprocessed_dataset\n",
|
|
|
+ "from llama_recipes.configs.datasets import samsum_dataset\n",
|
|
|
"\n",
|
|
|
"train_dataset = get_preprocessed_dataset(tokenizer, samsum_dataset, 'train')"
|
|
|
]
|