Skip to content

vllm.multimodal.cache

MultiModalCacheValue module-attribute 

MultiModalCacheValue = Union[
    MultiModalKwargsItems,
    MultiModalKwargsItem,
    MultiModalKwargs,
    Mapping[str, NestedTensors],
    MultiModalCacheItemMetadata,
]

_V module-attribute 

_V = TypeVar('_V', bound=MultiModalCacheValue)

logger module-attribute 

logger = init_logger(__name__)

MultiModalCache

Source code in vllm/multimodal/cache.py 
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
class MultiModalCache:

    @classmethod
    def get_leaf_size(
        cls,
        leaf: object,
        *,
        debug: bool = False,
    ) -> int:
        if isinstance(leaf, MultiModalFieldElem):
            return cls.get_item_size(leaf.data)  # type: ignore

        # These are not subclasses of dict
        if isinstance(leaf, MultiModalKwargsItems):
            return cls.get_item_size(leaf.data)  # type: ignore
        if isinstance(leaf, MultiModalKwargsItem):
            return cls.get_item_size(leaf.data)  # type: ignore
        if isinstance(leaf, MultiModalKwargs):
            return cls.get_item_size(leaf.data)  # type: ignore

        # sys.getsizeof doesn't work for tensors
        if isinstance(leaf, torch.Tensor):
            return leaf.nbytes

        if isinstance(leaf, MultiModalCacheItemMetadata):
            return leaf.size

        return sys.getsizeof(leaf)

    @classmethod
    def get_item_size(
        cls,
        value: MultiModalCacheValue,
        *,
        debug: bool = False,
    ) -> int:
        size = json_reduce_leaves(
            lambda a, b: a + b,
            json_map_leaves(lambda x: cls.get_leaf_size(x, debug=debug),
                            value),
        )

        if debug:
            logger.debug("Calculated size of %s to be %.2f GiB", type(value),
                         size / GiB_bytes)

        return size

    @classmethod
    def get_lru_cache(
        cls,
        capacity_gb: float,
        value_type: type[_V],
        *,
        debug: bool = False,
    ) -> LRUCache[str, _V]:
        return LRUCache(
            GiB_bytes * capacity_gb,
            getsizeof=lambda x: cls.get_item_size(x, debug=debug),
        )

get_item_size classmethod 

get_item_size(
    value: MultiModalCacheValue, *, debug: bool = False
) -> int
Source code in vllm/multimodal/cache.py 
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
@classmethod
def get_item_size(
    cls,
    value: MultiModalCacheValue,
    *,
    debug: bool = False,
) -> int:
    size = json_reduce_leaves(
        lambda a, b: a + b,
        json_map_leaves(lambda x: cls.get_leaf_size(x, debug=debug),
                        value),
    )

    if debug:
        logger.debug("Calculated size of %s to be %.2f GiB", type(value),
                     size / GiB_bytes)

    return size

get_leaf_size classmethod 

get_leaf_size(leaf: object, *, debug: bool = False) -> int
Source code in vllm/multimodal/cache.py 
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
@classmethod
def get_leaf_size(
    cls,
    leaf: object,
    *,
    debug: bool = False,
) -> int:
    if isinstance(leaf, MultiModalFieldElem):
        return cls.get_item_size(leaf.data)  # type: ignore

    # These are not subclasses of dict
    if isinstance(leaf, MultiModalKwargsItems):
        return cls.get_item_size(leaf.data)  # type: ignore
    if isinstance(leaf, MultiModalKwargsItem):
        return cls.get_item_size(leaf.data)  # type: ignore
    if isinstance(leaf, MultiModalKwargs):
        return cls.get_item_size(leaf.data)  # type: ignore

    # sys.getsizeof doesn't work for tensors
    if isinstance(leaf, torch.Tensor):
        return leaf.nbytes

    if isinstance(leaf, MultiModalCacheItemMetadata):
        return leaf.size

    return sys.getsizeof(leaf)

get_lru_cache classmethod 

get_lru_cache(
    capacity_gb: float,
    value_type: type[_V],
    *,
    debug: bool = False,
) -> LRUCache[str, _V]
Source code in vllm/multimodal/cache.py 
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
@classmethod
def get_lru_cache(
    cls,
    capacity_gb: float,
    value_type: type[_V],
    *,
    debug: bool = False,
) -> LRUCache[str, _V]:
    return LRUCache(
        GiB_bytes * capacity_gb,
        getsizeof=lambda x: cls.get_item_size(x, debug=debug),
    )

MultiModalCacheItemMetadata dataclass

Source code in vllm/multimodal/cache.py 
21
22
23
24
25
26
27
@dataclass
class MultiModalCacheItemMetadata:
    size: int

    @classmethod
    def wraps(cls, value: "MultiModalCacheValue"):
        return cls(size=MultiModalCache.get_item_size(value))

size instance-attribute 

size: int

__init__ 

__init__(size: int) -> None

wraps classmethod 

wraps(value: MultiModalCacheValue)
Source code in vllm/multimodal/cache.py 
25
26
27
@classmethod
def wraps(cls, value: "MultiModalCacheValue"):
    return cls(size=MultiModalCache.get_item_size(value))