Run a language model from C

// Text generation with the Kwker Runtime from C: load a Hugging Face model folder (weights and tokenizer), then continue a
// prompt until an end-of-sequence token. No Python, no PyTorch.
//
//   rt_generate <model folder> <precision> <new tokens> <prompt>
//
// precision is "int4", "int8", "bf16" or "float32". The program prints the generated text - at most <new tokens> tokens.
// Greedy by default; set KWKER_TEMPERATURE (and optionally KWKER_TOP_P, KWKER_SEED) to sample instead.
//
// Build: cc rt_generate.c -I<kwker include dir> -L<lib dir> -lkwker_rt -o rt_generate
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "kwker_rt.h"

static double env_or(const char* name, double fallback) {
    const char* v = getenv(name);
    return v ? atof(v) : fallback;
}

int main(int argc, char** argv) {
    if (argc != 5) {
        fprintf(stderr, "usage: %s <model folder> <precision> <new tokens> <prompt>\n", argv[0]);
        return 2;
    }
    const char* dir = argv[1];
    const char* precision = argv[2];
    const int64_t n_new = atoll(argv[3]);
    const char* text = argv[4];

    // The tokenizer: the prompt's text as token ids, with the model's start token if it uses one.
    kwker_rt_tokenizer* tok = NULL;
    if (kwker_rt_tokenizer_load(dir, &tok) != 0) {
        fprintf(stderr, "tokenizer: %s\n", kwker_rt_tokenizer_error());
        return 1;
    }
    int64_t n_prompt = 0;
    kwker_rt_tokenizer_encode(tok, text, -1, 1, NULL, 0, &n_prompt);
    int64_t* prompt = malloc(sizeof(int64_t) * (n_prompt > 0 ? n_prompt : 1));
    kwker_rt_tokenizer_encode(tok, text, -1, 1, prompt, n_prompt, &n_prompt);

    // The model: room for the prompt plus the new tokens, one sequence.
    kwker_rt_decoder* d = NULL;
    if (kwker_rt_decoder_load(dir, precision, n_prompt + n_new, 1, &d) != 0) {
        fprintf(stderr, "load failed: %s\n", kwker_rt_decoder_error());
        return 1;
    }

    // The model's end-of-sequence ids: generation stops at the first one.
    int64_t eos[8];
    int64_t n_eos = kwker_rt_decoder_eos(d, eos, 8);
    if (n_eos > 8) n_eos = 8;

    // Temperature 0 is greedy decoding; top-k 0, top-p 1, min-p 0 and repetition penalty 1 turn those off.
    const double temperature = env_or("KWKER_TEMPERATURE", 0.0);
    const double top_p = env_or("KWKER_TOP_P", 1.0);
    const uint64_t seed = (uint64_t)env_or("KWKER_SEED", 0);

    int64_t* out = malloc(sizeof(int64_t) * (n_new > 0 ? n_new : 1));
    int64_t n_out = 0;
    if (kwker_rt_decoder_generate(d, prompt, n_prompt, n_new, eos, n_eos, 0, temperature, 0, top_p, 0.0, 1.0, seed, out, &n_out) != 0) {
        fprintf(stderr, "generate failed: %s\n", kwker_rt_decoder_error());
        return 1;
    }

    // The new tokens as text, without special tokens such as the end-of-sequence one.
    int64_t len = 0;
    kwker_rt_tokenizer_decode(tok, out, n_out, 1, NULL, 0, &len);
    char* reply = malloc((size_t)len + 1);
    kwker_rt_tokenizer_decode(tok, out, n_out, 1, reply, len + 1, &len);
    printf("%s\n", reply);

    kwker_rt_decoder_free(d);
    kwker_rt_tokenizer_free(tok);
    free(reply);
    free(out);
    free(prompt);
    return 0;
}

cc rt_generate.c -I<include dir> -L<library dir> -lkwker_rt -Wl,-rpath,<library dir> -o rt_generate
./rt_generate ~/models/SmolLM2-135M int4 16 "The capital of France is"

 the city of Paris. It is the largest city in France and the capital of

KWKER_TEMPERATURE=0.8 KWKER_TOP_P=0.9 KWKER_SEED=42 ./rt_generate ~/models/SmolLM2-135M int4 16 "The capital of France is"

 the name of the palace. It is also the seat of the president of the

// Chat with an instruct model through the Kwker Runtime from C: the question (and an optional system prompt) become the
// model's prompt through its own chat template, the model writes its reply. No Python, no PyTorch.
//
//   rt_chat <model folder> <precision> <new tokens> <question> [<system prompt>]
//
// precision is "int4", "int8", "bf16" or "float32". The program prints the reply - at most <new tokens> tokens, greedy.
//
// Build: cc rt_chat.c -I<kwker include dir> -L<lib dir> -lkwker_rt -o rt_chat
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "kwker_rt.h"

// Append s to out as a JSON string (quotes included); out has room for 6 bytes per byte of s plus 3.
static char* json_string(char* out, const char* s) {
    *out++ = '"';
    for (; *s; s++) {
        unsigned char c = (unsigned char)*s;
        if (c == '"' || c == '\\') {
            *out++ = '\\';
            *out++ = (char)c;
        } else if (c < 0x20) {
            out += sprintf(out, "\\u%04x", c);
        } else {
            *out++ = (char)c;
        }
    }
    *out++ = '"';
    *out = 0;
    return out;
}

int main(int argc, char** argv) {
    if (argc != 5 && argc != 6) {
        fprintf(stderr, "usage: %s <model folder> <precision> <new tokens> <question> [<system prompt>]\n", argv[0]);
        return 2;
    }
    const char* dir = argv[1];
    const char* precision = argv[2];
    const int64_t n_new = atoll(argv[3]);
    const char* question = argv[4];
    const char* system = argc == 6 ? argv[5] : NULL;

    // The conversation as a JSON array of messages: [{"role": "system", ...}, {"role": "user", "content": "..."}].
    char* messages = malloc(64 + 6 * (strlen(question) + (system ? strlen(system) : 0)));
    char* p = messages;
    p += sprintf(p, "[");
    if (system) {
        p += sprintf(p, "{\"role\": \"system\", \"content\": ");
        p = json_string(p, system);
        p += sprintf(p, "}, ");
    }
    p += sprintf(p, "{\"role\": \"user\", \"content\": ");
    p = json_string(p, question);
    sprintf(p, "}]");

    kwker_rt_tokenizer* tok = NULL;
    if (kwker_rt_tokenizer_load(dir, &tok) != 0) {
        fprintf(stderr, "tokenizer: %s\n", kwker_rt_tokenizer_error());
        return 1;
    }

    // The prompt text from the model's chat template, ending where the assistant's reply starts.
    int64_t len = 0;
    if (kwker_rt_tokenizer_chat(tok, messages, 1, NULL, NULL, 0, &len) != 0) {
        fprintf(stderr, "chat template: %s\n", kwker_rt_tokenizer_error());
        return 1;
    }
    char* text = malloc((size_t)len + 1);
    kwker_rt_tokenizer_chat(tok, messages, 1, NULL, text, len + 1, &len);

    // The prompt's token ids. The template already holds the model's special tokens, so none are added.
    int64_t n_prompt = 0;
    kwker_rt_tokenizer_encode(tok, text, len, 0, NULL, 0, &n_prompt);
    int64_t* prompt = malloc(sizeof(int64_t) * (n_prompt > 0 ? n_prompt : 1));
    kwker_rt_tokenizer_encode(tok, text, len, 0, prompt, n_prompt, &n_prompt);

    kwker_rt_decoder* d = NULL;
    if (kwker_rt_decoder_load(dir, precision, n_prompt + n_new, 1, &d) != 0) {
        fprintf(stderr, "load failed: %s\n", kwker_rt_decoder_error());
        return 1;
    }
    int64_t eos[8];
    int64_t n_eos = kwker_rt_decoder_eos(d, eos, 8);
    if (n_eos > 8) n_eos = 8;

    // Greedy: temperature 0, the sampling settings off.
    int64_t* out = malloc(sizeof(int64_t) * (n_new > 0 ? n_new : 1));
    int64_t n_out = 0;
    if (kwker_rt_decoder_generate(d, prompt, n_prompt, n_new, eos, n_eos, 0, 0.0, 0, 1.0, 0.0, 1.0, 0, out, &n_out) != 0) {
        fprintf(stderr, "generate failed: %s\n", kwker_rt_decoder_error());
        return 1;
    }

    // The reply as text, without the end-of-turn token.
    kwker_rt_tokenizer_decode(tok, out, n_out, 1, NULL, 0, &len);
    char* reply = malloc((size_t)len + 1);
    kwker_rt_tokenizer_decode(tok, out, n_out, 1, reply, len + 1, &len);
    printf("%s\n", reply);

    kwker_rt_decoder_free(d);
    kwker_rt_tokenizer_free(tok);
    free(reply);
    free(out);
    free(prompt);
    free(text);
    free(messages);
    return 0;
}

cc rt_chat.c -I<include dir> -L<library dir> -lkwker_rt -Wl,-rpath,<library dir> -o rt_chat
./rt_chat ~/models/SmolLM2-135M-Instruct int8 24 "What is the capital of France?"

The capital of France is Paris.