Class EngineSettings

Namespace

Uralstech.UAI.LiteRTLM

public sealed class EngineSettings : LiteRTLMNativeHandle

Inheritance

object

LiteRTLMNativeHandle

EngineSettings

Inherited Members

LiteRTLMNativeHandle.Dispose()

Constructors

EngineSettings(int, string, string?, string?)

Creates managed engine settings from a raw file descriptor. The engine takes ownership of the file descriptor and closes it when done. The caller is responsible for disposing the wrapper using Dispose().

public EngineSettings(int fd, string backend, string? visionBackend = null, string? audioBackend = null)

Parameters

fd int

The file descriptor of the model.

backend string

The backend to use (for example, "cpu" or "gpu").

visionBackend string

The vision backend to use, or null if not set.

audioBackend string

The audio backend to use, or null if not set.

Remarks

See BackendNames for valid backend strings.

Exceptions

InvalidOperationException

Thrown if the native object could not be created.

EngineSettings(string, string, string?, string?)

Creates managed engine settings from a model path. The caller is responsible for disposing the wrapper using Dispose().

public EngineSettings(string modelPath, string backend, string? visionBackend = null, string? audioBackend = null)

Parameters

modelPath string

The path to the model file.

backend string

The backend to use (for example, "cpu" or "gpu").

visionBackend string

The vision backend to use, or null if not set.

audioBackend string

The audio backend to use, or null if not set.

Remarks

See BackendNames for valid backend strings.

Exceptions

InvalidOperationException

Thrown if the native object could not be created.

Methods

EnableBenchmark()

Enables benchmarking for the engine.

public void EnableBenchmark()

ReleaseUnmanagedResources()

protected override void ReleaseUnmanagedResources()

SetActivationDataType(ActivationDataType)

Sets the activation data type.

public void SetActivationDataType(ActivationDataType activationDataType)

Parameters

activationDataType ActivationDataType

The activation data type.

SetAudioLoraRank(int)

Sets the Audio LoRA rank for the engine.

public void SetAudioLoraRank(int loraRank)

Parameters

loraRank int

The Audio LoRA rank.

SetAudioNumThreads(int)

Sets the number of threads for the audio CPU backend.

public void SetAudioNumThreads(int numThreads)

Parameters

numThreads int

The number of threads.

SetCacheDir(string)

Sets the cache directory for the engine.

public void SetCacheDir(string cacheDir)

Parameters

cacheDir string

The cache directory.

SetEnableSpeculativeDecoding(bool)

Sets whether to enable speculative decoding.

public void SetEnableSpeculativeDecoding(bool enableSpeculativeDecoding)

Parameters

enableSpeculativeDecoding bool

Whether to enable speculative decoding.

SetLitertDispatchLibDir(string)

Sets the LiteRT dispatch library directory for the NPU backend.

public void SetLitertDispatchLibDir(string libDir)

Parameters

libDir string

The dispatch library directory.

SetLoraRank(int)

Sets the LoRA rank for the engine.

public void SetLoraRank(int loraRank)

Parameters

loraRank int

The LoRA rank.

SetMaxNumImages(int)

Sets the maximum number of images for the engine. This is only used for the legacy implementation of the engine.

public void SetMaxNumImages(int maxNumImages)

Parameters

maxNumImages int

The maximum number of images.

SetMaxNumTokens(int)

Sets the maximum number of tokens for the engine.

public void SetMaxNumTokens(int maxNumTokens)

Parameters

maxNumTokens int

The maximum number of tokens.

SetNumDecodeTokens(int)

Sets the number of decode tokens for benchmarking.

public void SetNumDecodeTokens(int numDecodeTokens)

Parameters

numDecodeTokens int

The number of decode tokens.

SetNumPrefillTokens(int)

Sets the number of prefill tokens for benchmarking.

public void SetNumPrefillTokens(int numPrefillTokens)

Parameters

numPrefillTokens int

The number of prefill tokens.

SetNumThreads(int)

Sets the number of threads for the CPU backend.

public void SetNumThreads(int numThreads)

Parameters

numThreads int

The number of threads.

SetParallelFileSectionLoading(bool)

Sets whether the engine should load different sections of the litertlm file in parallel. Defaults to true.

public void SetParallelFileSectionLoading(bool parallelFileSectionLoading)

Parameters

parallelFileSectionLoading bool

Whether to load in parallel.

SetPrefillChunkSize(int)

Sets the prefill chunk size for the engine. Only applicable for the CPU backend with dynamic models.

public void SetPrefillChunkSize(int prefillChunkSize)

Parameters

prefillChunkSize int

The prefill chunk size.

SetSupportedAudioLoraRanks(int[])

Sets the supported Audio LoRA ranks for the engine.

public int SetSupportedAudioLoraRanks(int[] loraRanks)

Parameters

loraRanks int[]

An array of supported Audio LoRA ranks.

Returns

int

0 on success, non-zero on failure.

SetSupportedLoraRanks(int[])

Sets the supported LoRA ranks for the engine.

public int SetSupportedLoraRanks(int[] loraRanks)

Parameters

loraRanks int[]

An array of supported LoRA ranks.

Returns

int

0 on success, non-zero on failure.