LLM Inference Overview

val model = ZeticMLangeLLMModel(context, PERSONAL_KEY, MODEL_NAME)

model.run("What is on-device AI?")

val sb = StringBuilder()
while (true) {
    val result = model.waitForNextToken()
    if (result.generatedTokens == 0) break
    sb.append(result.token)
}

val output = sb.toString()

let model = try ZeticMLangeLLMModel(personalKey: PERSONAL_KEY, name: MODEL_NAME)

try model.run("What is on-device AI?")

var buffer = ""
while true {
    let result = model.waitForNextToken()
    if result.generatedTokens == 0 { break }
    buffer.append(result.token)
}

let output = buffer

final model = await ZeticMLangeLLMModel.create(
  personalKey: personalKey,
  name: modelName,
);

model.run('What is on-device AI?');

final buffer = StringBuffer();
while (true) {
  final result = model.waitForNextToken();
  if (result.isFinished) {
    break;
  }
  buffer.write(result.token);
}

final output = buffer.toString();

val model = ZeticMLangeLLMModel(
    context = context,
    personalKey = PERSONAL_KEY,
    name = MODEL_NAME,
    target = LLMTarget.LLAMA_CPP,
    quantType = LLMQuantType.GGUF_QUANT_Q4_K_M,
    apType = APType.GPU,
    initOption = LLMInitOption(
        kvCacheCleanupPolicy = LLMKVCacheCleanupPolicy.DO_NOT_CLEAN_UP,
        nCtx = 4096,
    ),
)

let model = try ZeticMLangeLLMModel(
    personalKey: PERSONAL_KEY,
    name: MODEL_NAME,
    target: .LLAMA_CPP,
    quantType: .GGUF_QUANT_Q4_K_M,
    apType: .GPU,
    initOption: LLMInitOption(
        kvCacheCleanupPolicy: .DO_NOT_CLEAN_UP,
        nCtx: 4096
    )
)

final model = await ZeticMLangeLLMModel.create(
  personalKey: personalKey,
  name: modelName,
  quantType: LLMQuantType.ggufQuantQ4KM,
  apType: APType.gpu,
  initOption: const LLMInitOption(
    kvCacheCleanupPolicy: LLMKVCacheCleanupPolicy.doNotCleanUp,
    nCtx: 4096,
  ),
);