LLM Inference Overview

val model = ZeticMLangeLLMModel(context, PERSONAL_KEY, MODEL_NAME)

model.run("What is on-device AI?")

val sb = StringBuilder()
while (true) {
    val result = model.waitForNextToken()
    if (result.generatedTokens == 0) break
    sb.append(result.token)
}

val output = sb.toString()

let model = try ZeticMLangeLLMModel(personalKey: PERSONAL_KEY, name: MODEL_NAME)

try model.run("What is on-device AI?")

var buffer = ""
while true {
    let result = model.waitForNextToken()
    if result.generatedTokens == 0 { break }
    buffer.append(result.token)
}

let output = buffer

val model = ZeticMLangeLLMModel(
    context = context,
    personalKey = PERSONAL_KEY,
    name = MODEL_NAME,
    target = LLMTarget.LLAMA_CPP,
    quantType = LLMQuantType.GGUF_QUANT_Q4_K_M,
    apType = APType.GPU,
    initOption = LLMInitOption(
        kvCacheCleanupPolicy = LLMKVCacheCleanupPolicy.DO_NOT_CLEAN_UP,
        nCtx = 4096,
    ),
)

let model = try ZeticMLangeLLMModel(
    personalKey: PERSONAL_KEY,
    name: MODEL_NAME,
    target: .LLAMA_CPP,
    quantType: .GGUF_QUANT_Q4_K_M,
    apType: .GPU,
    initOption: LLMInitOption(
        kvCacheCleanupPolicy: .DO_NOT_CLEAN_UP,
        nCtx: 4096
    )
)