import torch
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline
)

base_model_name = "TheBloke/leo-hessianai-7B-GPTQ"
#base_model_name = "LeoLM/leo-hessianai-7b"
# Variante des Modells
revision = "gptq-8bit-32g-actorder_True" 
#revision = "main"

refined_model = "german_quotes"

# Quantization Config
quant_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.float16,
    bnb_4bit_use_double_quant=False
)

# Model
base_model = AutoModelForCausalLM.from_pretrained(
    base_model_name,
    revision=revision,
    #quantization_config=quant_config,
    #device_map={"": 0}  # uncomment for GPU
)

tokenizer = AutoTokenizer.from_pretrained(base_model_name, 
                                          trust_remote_code=False)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "right"  # Fix for fp16

text_gen = pipeline(task="text-generation", model=base_model,
                    max_length=200, tokenizer=tokenizer)

system_prompt = """Dies ist eine Unterhaltung zwischen \
einem intelligenten, hilfsbereitem \
KI-Assistenten und einem Nutzer.
Der Assistent gibt Antworten in Form von Zitaten."""
prompt_format = "<|im_start|>system\n{system_prompt}\
<|im_end|>\n<|im_start|>user\nZitiere {prompt}\
<|im_end|>\n<|im_start|>assistant\n"

def get_quote(author:str, max_length:int=200):
    query = prompt_format.format(system_prompt=system_prompt, prompt= author)
    output = text_gen(query, do_sample=True, top_p=0.95, max_length=max_length, 
                      return_full_text=False, pad_token_id=tokenizer.pad_token_id)
    print(output[0]['generated_text'])

base_model.load_adapter(refined_model, adapter_name=refined_model)
# Lädt das Lora Modell in den Speicher
base_model.enable_adapters() # Aktiviert das LORA Modell

get_quote("Christoph")
get_quote("Oscar Wilde")
get_quote("Aristoteles")
get_quote("Heinrich Heine")

base_model.disable_adapters() # Deaktiviert das LORA Modell
get_quote("Christoph")
get_quote("Oscar Wilde")
get_quote("Aristoteles")
get_quote("Heinrich Heine")

import torch
from ctransformers import (
    AutoModelForCausalLM,
)
from transformers import pipeline

base_model = AutoModelForCausalLM.from_pretrained(
    "caretech-owl/leo-hessionai-7B-quotes-gguf", model_type="llama")

def get_quote(author:str, max_new_tokens:int=200):
    query = prompt_format.format(system_prompt=system_prompt, prompt= author)
    output = base_model(query, stop='<|im_end|>', max_new_tokens=max_new_tokens)
    print(output)

get_quote("Christoph")
get_quote("Oscar Wilde")
get_quote("Aristoteles")
get_quote("Heinrich Heine")

LLM mit LoRA verwenden¶

Am Beispiel deutscher Zitate¶

Vorbereiten des Grundmodells¶

Vorbereiten der Textpipeline¶

Anfragen absetzen¶

Alternative¶