Class NativeAPI.EngineSettings

Namespace

Uralstech.UAI.LiteRTLM.Native

public static class NativeAPI.EngineSettings

Inheritance

object

NativeAPI.EngineSettings

Methods

litert_lm_engine_settings_create(string, string, string?, string?)

Creates LiteRT LM Engine Settings. The caller is responsible for destroying the settings using litert_lm_engine_settings_delete(nint).

public static extern nint litert_lm_engine_settings_create(string modelPath, string backendStr, string? visionBackendStr, string? audioBackendStr)

Parameters

modelPath string

The path to the model file.

backendStr string

The backend to use (e.g., "cpu", "gpu").

visionBackendStr string

The vision backend to use, or null if not set.

audioBackendStr string

The audio backend to use, or null if not set.

Returns

nint

A pointer to the created settings, or nint.Zero on failure.

litert_lm_engine_settings_create_from_raw_file_descriptor(int, string, string?, string?)

Creates LiteRT LM Engine Settings from a raw file descriptor. The engine takes ownership of the file descriptor and will close it when done. The caller is responsible for destroying the settings using litert_lm_engine_settings_delete(nint).

public static extern nint litert_lm_engine_settings_create_from_raw_file_descriptor(int fd, string backendStr, string? visionBackendStr, string? audioBackendStr)

Parameters

fd int

The file descriptor of the model.

backendStr string

The backend to use (e.g., "cpu", "gpu").

visionBackendStr string

The vision backend to use, or null if not set.

audioBackendStr string

The audio backend to use, or null if not set.

Returns

nint

A pointer to the created settings, or nint.Zero on failure.

litert_lm_engine_settings_delete(nint)

Destroys LiteRT LM Engine Settings.

public static extern void litert_lm_engine_settings_delete(nint settings)

Parameters

settings nint

The settings to destroy.

litert_lm_engine_settings_enable_benchmark(nint)

Enables benchmarking for the engine.

public static extern void litert_lm_engine_settings_enable_benchmark(nint settings)

Parameters

settings nint

The engine settings.

litert_lm_engine_settings_set_activation_data_type(nint, ActivationDataType)

Sets the activation data type.

public static extern void litert_lm_engine_settings_set_activation_data_type(nint settings, ActivationDataType activationDataType)

Parameters

settings nint

The engine settings.

activationDataType ActivationDataType

The activation data type.

litert_lm_engine_settings_set_audio_lora_rank(nint, int)

Sets the Audio LoRA rank for the engine.

public static extern void litert_lm_engine_settings_set_audio_lora_rank(nint settings, int loraRank)

Parameters

settings nint

The engine settings.

loraRank int

The Audio LoRA rank.

litert_lm_engine_settings_set_audio_num_threads(nint, int)

Sets the number of threads for the audio CPU backend.

public static extern void litert_lm_engine_settings_set_audio_num_threads(nint settings, int numThreads)

Parameters

settings nint

The engine settings.

numThreads int

The number of threads.

litert_lm_engine_settings_set_cache_dir(nint, string)

Sets the cache directory for the engine.

public static extern void litert_lm_engine_settings_set_cache_dir(nint settings, string cacheDir)

Parameters

settings nint

The engine settings.

cacheDir string

The cache directory.

litert_lm_engine_settings_set_enable_speculative_decoding(nint, bool)

Sets whether to enable speculative decoding.

public static extern void litert_lm_engine_settings_set_enable_speculative_decoding(nint settings, bool enableSpeculativeDecoding)

Parameters

settings nint

The engine settings.

enableSpeculativeDecoding bool

Whether to enable speculative decoding.

litert_lm_engine_settings_set_litert_dispatch_lib_dir(nint, string)

Sets the LiteRT dispatch library directory for NPU backend.

public static extern void litert_lm_engine_settings_set_litert_dispatch_lib_dir(nint settings, string libDir)

Parameters

settings nint

The engine settings.

libDir string

The dispatch library directory.

litert_lm_engine_settings_set_lora_rank(nint, int)

Sets the LoRA rank for the engine.

public static extern void litert_lm_engine_settings_set_lora_rank(nint settings, int loraRank)

Parameters

settings nint

The engine settings.

loraRank int

The LoRA rank.

litert_lm_engine_settings_set_max_num_images(nint, int)

Sets the maximum number of images for the engine. This is only used for the legacy implementation of the engine.

public static extern void litert_lm_engine_settings_set_max_num_images(nint settings, int maxNumImages)

Parameters

settings nint

The engine settings.

maxNumImages int

The maximum number of images.

litert_lm_engine_settings_set_max_num_tokens(nint, int)

Sets the maximum number of tokens for the engine.

public static extern void litert_lm_engine_settings_set_max_num_tokens(nint settings, int maxNumTokens)

Parameters

settings nint

The engine settings.

maxNumTokens int

The maximum number of tokens.

litert_lm_engine_settings_set_num_decode_tokens(nint, int)

Sets the number of decode tokens for benchmarking.

public static extern void litert_lm_engine_settings_set_num_decode_tokens(nint settings, int numDecodeTokens)

Parameters

settings nint

The engine settings.

numDecodeTokens int

The number of decode tokens.

litert_lm_engine_settings_set_num_prefill_tokens(nint, int)

Sets the number of prefill tokens for benchmarking.

public static extern void litert_lm_engine_settings_set_num_prefill_tokens(nint settings, int numPrefillTokens)

Parameters

settings nint

The engine settings.

numPrefillTokens int

The number of prefill tokens.

litert_lm_engine_settings_set_num_threads(nint, int)

Sets the number of threads for the CPU backend.

public static extern void litert_lm_engine_settings_set_num_threads(nint settings, int numThreads)

Parameters

settings nint

The engine settings.

numThreads int

The number of threads.

litert_lm_engine_settings_set_parallel_file_section_loading(nint, bool)

Sets whether the engine should load different sections of the litertlm file in parallel. Defaults to true.

public static extern void litert_lm_engine_settings_set_parallel_file_section_loading(nint settings, bool parallelFileSectionLoading)

Parameters

settings nint

The engine settings.

parallelFileSectionLoading bool

Whether to load in parallel.

litert_lm_engine_settings_set_prefill_chunk_size(nint, int)

Sets the prefill chunk size for the engine. Only applicable for CPU backend with dynamic models.

public static extern void litert_lm_engine_settings_set_prefill_chunk_size(nint settings, int prefillChunkSize)

Parameters

settings nint

The engine settings.

prefillChunkSize int

The prefill chunk size.

litert_lm_engine_settings_set_supported_audio_lora_ranks(nint, int[], nuint)

Sets the supported Audio LoRA ranks for the engine.

public static extern int litert_lm_engine_settings_set_supported_audio_lora_ranks(nint settings, int[] loraRanks, nuint numRanks)

Parameters

settings nint

The engine settings.

loraRanks int[]

An array of supported Audio LoRA ranks.

numRanks nuint

The number of ranks in the array.

Returns

int

0 on success, non-zero on failure.

litert_lm_engine_settings_set_supported_lora_ranks(nint, int[], nuint)

Sets the supported LoRA ranks for the engine.

public static extern int litert_lm_engine_settings_set_supported_lora_ranks(nint settings, int[] loraRanks, nuint numRanks)

Parameters

settings nint

The engine settings.

loraRanks int[]

An array of supported LoRA ranks.

numRanks nuint

The number of ranks in the array.

Returns

int

0 on success, non-zero on failure.