Skip to content

vllm.model_executor.layers.quantization.inc

INCConfig

Bases: QuantizationConfig

Config class for FP8 using Intel Neural Compressor.

Source code in vllm/model_executor/layers/quantization/inc.py 
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
class INCConfig(QuantizationConfig):
    """Config class for FP8 using Intel Neural Compressor."""

    @classmethod
    def get_name(cls) -> QuantizationMethods:
        return "inc"

    @classmethod
    def get_supported_act_dtypes(cls) -> list[torch.dtype]:
        return [torch.bfloat16]

    @classmethod
    def from_config(cls, config: dict[str, Any]) -> "INCConfig":
        raise AssertionError

    def get_quant_method(self, layer: torch.nn.Module,
                         prefix: str) -> Optional["QuantizeMethodBase"]:
        if isinstance(layer, LinearBase):
            return UnquantizedLinearMethod()
        elif isinstance(layer, FusedMoE):
            return UnquantizedFusedMoEMethod(layer.moe_config)
        return None

    @classmethod
    def get_min_capability(cls) -> int:
        raise AssertionError

    @staticmethod
    def get_config_filenames() -> list[str]:
        return []

from_config classmethod 

from_config(config: dict[str, Any]) -> INCConfig
Source code in vllm/model_executor/layers/quantization/inc.py 
43
44
45
@classmethod
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
    raise AssertionError

get_config_filenames staticmethod 

get_config_filenames() -> list[str]
Source code in vllm/model_executor/layers/quantization/inc.py 
59
60
61
@staticmethod
def get_config_filenames() -> list[str]:
    return []

get_min_capability classmethod 

get_min_capability() -> int
Source code in vllm/model_executor/layers/quantization/inc.py 
55
56
57
@classmethod
def get_min_capability(cls) -> int:
    raise AssertionError

get_name classmethod 

get_name() -> QuantizationMethods
Source code in vllm/model_executor/layers/quantization/inc.py 
35
36
37
@classmethod
def get_name(cls) -> QuantizationMethods:
    return "inc"

get_quant_method 

get_quant_method(
    layer: Module, prefix: str
) -> Optional[QuantizeMethodBase]
Source code in vllm/model_executor/layers/quantization/inc.py 
47
48
49
50
51
52
53
def get_quant_method(self, layer: torch.nn.Module,
                     prefix: str) -> Optional["QuantizeMethodBase"]:
    if isinstance(layer, LinearBase):
        return UnquantizedLinearMethod()
    elif isinstance(layer, FusedMoE):
        return UnquantizedFusedMoEMethod(layer.moe_config)
    return None

get_supported_act_dtypes classmethod 

get_supported_act_dtypes() -> list[dtype]
Source code in vllm/model_executor/layers/quantization/inc.py 
39
40
41
@classmethod
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
    return [torch.bfloat16]