gerd.frontends.instruct

A gradio frontend to interact with the GERD instruct service.

Classes:

Name	Description
Global	Singleton to store the service.

Functions:

Name	Description
generate	Generate text from the model.
load_model	Load a global large language model.

Attributes:

Name	Type	Description
KIOSK_MODE		Whether the frontend is running in kiosk mode.

KIOSK_MODE module-attribute

KIOSK_MODE = kiosk_mode

Whether the frontend is running in kiosk mode.

Kiosk mode reduces the number of options to a minimum and automatically loads the model.

Global

Singleton to store the service.

generate

generate(temperature: float, top_p: float, max_tokens: int, system_text: str, *args: str) -> str

Generate text from the model.

Parameters:

Name	Type	Description	Default
temperature	float	The temperature for the generation	required
top_p	float	The top-p value for the generation	required
max_tokens	int	The maximum number of tokens to generate	required
system_text	str	The system text to set up the context	required
args	str	The user input	()

Source code in gerd/frontends/instruct.py

def generate(
    temperature: float,
    top_p: float,
    max_tokens: int,
    system_text: str,
    *args: str,
) -> str:
    """Generate text from the model.

    Parameters:
        temperature: The temperature for the generation
        top_p: The top-p value for the generation
        max_tokens: The maximum number of tokens to generate
        system_text: The system text to set up the context
        args: The user input
    """
    fields = dict(zip(config.model.prompt_config.parameters, args, strict=True))
    if Global.service is None:
        msg = "Model not loaded"
        raise gr.Error(msg)
    if system_text:
        Global.service.config.model.prompt_setup = [
            ("system", PromptConfig(text=system_text))
        ]
    Global.service.config.model.top_p = top_p
    Global.service.config.model.temperature = temperature
    Global.service.config.model.max_new_tokens = max_tokens
    Global.service.reset()
    return Global.service.submit_user_message(fields).text

load_model

load_model(model_name: str, origin: str = 'None') -> dict[str, Any]

Load a global large language model.

Parameters:

Name	Type	Description	Default
model_name	str	The name of the model	required
origin	str	Whether to use an extra LoRA	'None'

Source code in gerd/frontends/instruct.py

def load_model(model_name: str, origin: str = "None") -> dict[str, Any]:
    """Load a global large language model.

    Parameters:
        model_name: The name of the model
        origin: Whether to use an extra LoRA
    """
    if Global.service is not None:
        _LOGGER.debug("Unloading model")
        del Global.service
    model_config = config.model_copy()
    model_config.model.name = model_name
    if origin != "None" and (lora_dir / origin).is_dir():
        model_config.model.loras.add(lora_dir / origin)
    _LOGGER.info("Loading model %s", model_config.model.name)
    Global.service = ChatService(model_config)
    _LOGGER.info("Model loaded")
    return gr.update(interactive=True)