In [ ]:

! pip install accelerate peft bitsandbytes transformers trl

! pip install accelerate peft bitsandbytes transformers trl

In [ ]:

from huggingface_hub import notebook_login
notebook_login()

from huggingface_hub import notebook_login notebook_login()

Model Training¶

In [ ]:

import torch
from datasets import load_dataset, Dataset
from peft import LoraConfig, AutoPeftModelForCausalLM
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TrainingArguments
from trl import SFTTrainer
import os

def finetune_llama_v2():
    data = load_dataset("code_x_glue_tc_nl_code_search_adv", split="validation")
    data_df = data.to_pandas()
    data_df["text"] = data_df[["docstring", "code"]].apply(lambda x: "<s>[INST] Docstring: " + x["docstring"] + " [/INST] Code: " + x["code"] + "</s>", axis=1)
    data = Dataset.from_pandas(data_df)
    tokenizer = AutoTokenizer.from_pretrained("codellama/CodeLlama-7b-hf")
    tokenizer.pad_token = tokenizer.eos_token
    bnb_config = BitsAndBytesConfig(
        load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype="float16", bnb_4bit_use_double_quant=True
    )
    model = AutoModelForCausalLM.from_pretrained(
        "codellama/CodeLlama-7b-hf", quantization_config=bnb_config, device_map="auto"
    )
    model.config.use_cache=False
    model.config.pretraining_tp=1
    peft_config = LoraConfig(
        r=8, lora_alpha=16, lora_dropout=0.05, bias="none", task_type="CAUSAL_LM"
    )
    training_arguments = TrainingArguments(
        output_dir="codellama2-finetuned-codex",
        per_device_train_batch_size=8,
        gradient_accumulation_steps=4,
        optim="paged_adamw_32bit",
        learning_rate=2e-4,
        lr_scheduler_type="cosine",
        save_strategy="epoch",
        logging_steps=10,
        num_train_epochs=1,
        max_steps=100,
        fp16=True,
        push_to_hub=True
    )
    trainer = SFTTrainer(
        model=model,
        train_dataset=data,
        peft_config=peft_config,
        dataset_text_field="text",
        args=training_arguments,
        tokenizer=tokenizer,
        packing=False,
        max_seq_length=512
    )
    trainer.train()
    trainer.push_to_hub()

if __name__ == "__main__":
    finetune_llama_v2()

import torch from datasets import load_dataset, Dataset from peft import LoraConfig, AutoPeftModelForCausalLM from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TrainingArguments from trl import SFTTrainer import os def finetune_llama_v2(): data = load_dataset("code_x_glue_tc_nl_code_search_adv", split="validation") data_df = data.to_pandas() data_df["text"] = data_df[["docstring", "code"]].apply(lambda x: "[INST] Docstring: " + x["docstring"] + " [/INST] Code: " + x["code"] + "", axis=1) data = Dataset.from_pandas(data_df) tokenizer = AutoTokenizer.from_pretrained("codellama/CodeLlama-7b-hf") tokenizer.pad_token = tokenizer.eos_token bnb_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype="float16", bnb_4bit_use_double_quant=True ) model = AutoModelForCausalLM.from_pretrained( "codellama/CodeLlama-7b-hf", quantization_config=bnb_config, device_map="auto" ) model.config.use_cache=False model.config.pretraining_tp=1 peft_config = LoraConfig( r=8, lora_alpha=16, lora_dropout=0.05, bias="none", task_type="CAUSAL_LM" ) training_arguments = TrainingArguments( output_dir="codellama2-finetuned-codex", per_device_train_batch_size=8, gradient_accumulation_steps=4, optim="paged_adamw_32bit", learning_rate=2e-4, lr_scheduler_type="cosine", save_strategy="epoch", logging_steps=10, num_train_epochs=1, max_steps=100, fp16=True, push_to_hub=True ) trainer = SFTTrainer( model=model, train_dataset=data, peft_config=peft_config, dataset_text_field="text", args=training_arguments, tokenizer=tokenizer, packing=False, max_seq_length=512 ) trainer.train() trainer.push_to_hub() if __name__ == "__main__": finetune_llama_v2()

Model Saving¶

In [ ]:

! cp -r /content/codellama2-finetuned-codex /content/drive/MyDrive/

! cp -r /content/codellama2-finetuned-codex /content/drive/MyDrive/

Model Push to Hub¶

In [ ]:

from peft import AutoPeftModelForCausalLM, PeftModel
from transformers import AutoModelForCausalLM
import torch
import os
model = AutoModelForCausalLM.from_pretrained(
            "codellama/CodeLlama-7b-hf", torch_dtype=torch.float16, load_in_8bit=False, device_map="auto", trust_remote_code=True)

from peft import AutoPeftModelForCausalLM, PeftModel from transformers import AutoModelForCausalLM import torch import os model = AutoModelForCausalLM.from_pretrained( "codellama/CodeLlama-7b-hf", torch_dtype=torch.float16, load_in_8bit=False, device_map="auto", trust_remote_code=True)

In [ ]:

peft_model = PeftModel.from_pretrained(model, "Vasanth/codellama2-finetuned-codex", from_transformers=True, device_map={"":0})

peft_model = PeftModel.from_pretrained(model, "Vasanth/codellama2-finetuned-codex", from_transformers=True, device_map={"":0})

In [ ]:

model = peft_model.merge_and_unload()

model = peft_model.merge_and_unload()

In [ ]:

model.push_to_hub("codellama2-finetuned-codex-fin")

model.push_to_hub("codellama2-finetuned-codex-fin")

In [ ]:

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("codellama/CodeLlama-7b-hf")
tokenizer.push_to_hub("codellama2-finetuned-codex-fin")

from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained("codellama/CodeLlama-7b-hf") tokenizer.push_to_hub("codellama2-finetuned-codex-fin")

Model Inferencing¶

In [ ]:

from transformers import AutoTokenizer
from transformers import pipeline
import torch

tokenizer = AutoTokenizer.from_pretrained("Vasanth/codellama2-finetuned-codex-fin")
pipe = pipeline(
    "text-generation",
    model="Vasanth/codellama2-finetuned-codex-fin",
    torch_dtype=torch.float16,
    device_map="auto",
)

sequences = pipe(
    'def fibonacci(',
    do_sample=True,
    temperature=0.2,
    top_p=0.9,
    num_return_sequences=1,
    eos_token_id=tokenizer.eos_token_id,
    max_length=100,
)
for seq in sequences:
    print(f"Result: {seq['generated_text']}")

from transformers import AutoTokenizer from transformers import pipeline import torch tokenizer = AutoTokenizer.from_pretrained("Vasanth/codellama2-finetuned-codex-fin") pipe = pipeline( "text-generation", model="Vasanth/codellama2-finetuned-codex-fin", torch_dtype=torch.float16, device_map="auto", ) sequences = pipe( 'def fibonacci(', do_sample=True, temperature=0.2, top_p=0.9, num_return_sequences=1, eos_token_id=tokenizer.eos_token_id, max_length=100, ) for seq in sequences: print(f"Result: {seq['generated_text']}")

In [ ]: