in optimum/commands/export/openvino.py [0:0]
def parse_args_openvino(parser: "ArgumentParser"):
required_group = parser.add_argument_group("Required arguments")
required_group.add_argument(
"-m", "--model", type=str, required=True, help="Model ID on huggingface.co or path on disk to load model from."
)
required_group.add_argument(
"output", type=Path, help="Path indicating the directory where to store the generated OV model."
)
optional_group = parser.add_argument_group("Optional arguments")
optional_group.add_argument(
"--task",
default="auto",
help=(
"The task to export the model for. If not specified, the task will be auto-inferred based on the model. Available tasks depend on the model, but are among:"
f" {str(TasksManager.get_all_tasks())}. For decoder models, use `xxx-with-past` to export the model using past key values in the decoder."
),
)
optional_group.add_argument(
"--framework",
type=str,
choices=["pt", "tf"],
default=None,
help=(
"The framework to use for the export. If not provided, will attempt to use the local checkpoint's original framework or what is available in the environment."
),
)
optional_group.add_argument(
"--trust-remote-code",
action="store_true",
help=(
"Allows to use custom code for the modeling hosted in the model repository. This option should only be set for repositories you trust and in which "
"you have read the code, as it will execute on your local machine arbitrary code present in the model repository."
),
)
optional_group.add_argument(
"--weight-format",
type=str,
choices=["fp32", "fp16", "int8", "int4", "mxfp4", "nf4"],
default=None,
help="The weight format of the exported model.",
)
optional_group.add_argument(
"--quant-mode",
type=str,
choices=["int8", "f8e4m3", "f8e5m2", "nf4_f8e4m3", "nf4_f8e5m2", "int4_f8e4m3", "int4_f8e5m2"],
default=None,
help=(
"Quantization precision mode. This is used for applying full model quantization including activations. "
),
)
optional_group.add_argument(
"--library",
type=str,
choices=["transformers", "diffusers", "timm", "sentence_transformers", "open_clip"],
default=None,
help="The library used to load the model before export. If not provided, will attempt to infer the local checkpoint's library",
)
optional_group.add_argument(
"--cache_dir",
type=str,
default=HUGGINGFACE_HUB_CACHE,
help="The path to a directory in which the downloaded model should be cached if the standard cache should not be used.",
)
optional_group.add_argument(
"--pad-token-id",
type=int,
default=None,
help=(
"This is needed by some models, for some tasks. If not provided, will attempt to use the tokenizer to guess it."
),
)
optional_group.add_argument(
"--variant",
type=str,
default=None,
help=("If specified load weights from variant filename."),
)
optional_group.add_argument(
"--ratio",
type=float,
default=None,
help=(
"A parameter used when applying 4-bit quantization to control the ratio between 4-bit and 8-bit quantization. If set to 0.8, 80%% of the layers will be quantized to int4 "
"while 20%% will be quantized to int8. This helps to achieve better accuracy at the sacrifice of the model size and inference latency. Default value is 1.0. "
"Note: If dataset is provided, and the ratio is less than 1.0, then data-aware mixed precision assignment will be applied."
),
)
optional_group.add_argument(
"--sym",
action="store_true",
default=None,
help=(
"Whether to apply symmetric quantization. This argument is related to integer-typed --weight-format and --quant-mode options. "
"In case of full or mixed quantization (--quant-mode) symmetric quantization will be applied to weights in any case, so only activation quantization "
"will be affected by --sym argument. For weight-only quantization (--weight-format) --sym argument does not affect backup precision. "
"Examples: (1) --weight-format int8 --sym => int8 symmetric quantization of weights; "
"(2) --weight-format int4 => int4 asymmetric quantization of weights; "
"(3) --weight-format int4 --sym --backup-precision int8_asym => int4 symmetric quantization of weights with int8 asymmetric backup precision; "
"(4) --quant-mode int8 --sym => weights and activations are quantized to int8 symmetric data type; "
"(5) --quant-mode int8 => activations are quantized to int8 asymmetric data type, weights -- to int8 symmetric data type; "
"(6) --quant-mode int4_f8e5m2 --sym => activations are quantized to f8e5m2 data type, weights -- to int4 symmetric data type."
),
)
optional_group.add_argument(
"--group-size",
type=int,
default=None,
help=("The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization."),
)
optional_group.add_argument(
"--backup-precision",
type=str,
choices=["none", "int8_sym", "int8_asym"],
default=None,
help=(
"Defines a backup precision for mixed-precision weight compression. Only valid for 4-bit weight formats. "
"If not provided, backup precision is int8_asym. 'none' stands for original floating-point precision of "
"the model weights, in this case weights are retained in their original precision without any "
"quantization. 'int8_sym' stands for 8-bit integer symmetric quantization without zero point. 'int8_asym' "
"stands for 8-bit integer asymmetric quantization with zero points per each quantization group."
),
)
optional_group.add_argument(
"--dataset",
type=str,
default=None,
help=(
"The dataset used for data-aware compression or quantization with NNCF. "
"For language models you can use the one from the list ['auto','wikitext2','c4','c4-new']. With 'auto' the "
"dataset will be collected from model's generations. "
"For diffusion models it should be on of ['conceptual_captions',"
"'laion/220k-GPT4Vision-captions-from-LIVIS','laion/filtered-wit']. "
"For visual language models the dataset must be set to 'contextual'. "
"Note: if none of the data-aware compression algorithms are selected and ratio parameter is omitted or "
"equals 1.0, the dataset argument will not have an effect on the resulting model."
),
)
optional_group.add_argument(
"--all-layers",
action="store_true",
default=None,
help=(
"Whether embeddings and last MatMul layers should be compressed to INT4. If not provided an weight "
"compression is applied, they are compressed to INT8."
),
)
optional_group.add_argument(
"--awq",
action="store_true",
default=None,
help=(
"Whether to apply AWQ algorithm. AWQ improves generation quality of INT4-compressed LLMs. If dataset is "
"provided, a data-aware activation-based version of the algorithm will be executed, which requires "
"additional time. Otherwise, data-free AWQ will be applied which relies on per-column magnitudes of "
"weights instead of activations. Note: it is possible that there will be no matching patterns in the model "
"to apply AWQ, in such case it will be skipped."
),
)
optional_group.add_argument(
"--scale-estimation",
action="store_true",
default=None,
help=(
"Indicates whether to apply a scale estimation algorithm that minimizes the L2 error between the original "
"and compressed layers. Providing a dataset is required to run scale estimation. Please note, that "
"applying scale estimation takes additional memory and time."
),
)
optional_group.add_argument(
"--gptq",
action="store_true",
default=None,
help=(
"Indicates whether to apply GPTQ algorithm that optimizes compressed weights in a layer-wise fashion to "
"minimize the difference between activations of a compressed and original layer. Please note, that "
"applying GPTQ takes additional memory and time."
),
)
optional_group.add_argument(
"--lora-correction",
action="store_true",
default=None,
help=(
"Indicates whether to apply LoRA Correction algorithm. When enabled, this algorithm introduces low-rank "
"adaptation layers in the model that can recover accuracy after weight compression at some cost of "
"inference latency. Please note, that applying LoRA Correction algorithm takes additional memory and time."
),
)
optional_group.add_argument(
"--sensitivity-metric",
type=str,
default=None,
help=(
"The sensitivity metric for assigning quantization precision to layers. It can be one of the following: "
"['weight_quantization_error', 'hessian_input_activation', 'mean_activation_variance', "
"'max_activation_variance', 'mean_activation_magnitude']."
),
)
optional_group.add_argument(
"--num-samples",
type=int,
default=None,
help="The maximum number of samples to take from the dataset for quantization.",
)
optional_group.add_argument(
"--disable-stateful",
action="store_true",
help=(
"Disable stateful converted models, stateless models will be generated instead. Stateful models are produced by default when this key is not used. "
"In stateful models all kv-cache inputs and outputs are hidden in the model and are not exposed as model inputs and outputs. "
"If --disable-stateful option is used, it may result in sub-optimal inference performance. "
"Use it when you intentionally want to use a stateless model, for example, to be compatible with existing "
"OpenVINO native inference code that expects KV-cache inputs and outputs in the model."
),
)
optional_group.add_argument(
"--disable-convert-tokenizer",
action="store_true",
help="Do not add converted tokenizer and detokenizer OpenVINO models.",
)
optional_group.add_argument(
"--smooth-quant-alpha",
type=float,
default=None,
help=(
"SmoothQuant alpha parameter that improves the distribution of activations before MatMul layers and "
"reduces quantization error. Valid only when activations quantization is enabled."
),
)
optional_group.add_argument(
"--model-kwargs",
type=json.loads,
help=("Any kwargs passed to the model forward, or used to customize the export for a given model."),
)