Skip to content

vllm.transformers_utils.runai_utils

SUPPORTED_SCHEMES module-attribute 

SUPPORTED_SCHEMES = ['s3://', 'gs://']

logger module-attribute 

logger = init_logger(__name__)

runai_model_streamer module-attribute 

runai_model_streamer = PlaceholderModule(
    "runai_model_streamer"
)

ObjectStorageModel

A class representing an ObjectStorage model mirrored into a temporary directory.

Attributes:

Name Type Description
dir

The temporary created directory.

Methods:

Name Description
pull_files

Pull model from object storage to the temporary directory.
Source code in vllm/transformers_utils/runai_utils.py 
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
class ObjectStorageModel:
    """
    A class representing an ObjectStorage model mirrored into a
    temporary directory.

    Attributes:
        dir: The temporary created directory.

    Methods:
        pull_files(): Pull model from object storage to the temporary directory.
    """

    def __init__(self, url: str) -> None:
        if envs.VLLM_ASSETS_CACHE_MODEL_CLEAN:
            for sig in (signal.SIGINT, signal.SIGTERM):
                existing_handler = signal.getsignal(sig)
                signal.signal(sig, self._close_by_signal(existing_handler))

        dir_name = os.path.join(
            get_cache_dir(), "model_streamer",
            hashlib.sha256(str(url).encode()).hexdigest()[:8])
        if os.path.exists(dir_name):
            shutil.rmtree(dir_name)
        os.makedirs(dir_name)
        self.dir = dir_name
        logger.debug("Init object storage, model cache path is: %s", dir_name)

    def _close(self) -> None:
        if os.path.exists(self.dir):
            shutil.rmtree(self.dir)

    def _close_by_signal(self, existing_handler=None):

        def new_handler(signum, frame):
            self._close()
            if existing_handler:
                existing_handler(signum, frame)

        return new_handler

    def pull_files(self,
                   model_path: str = "",
                   allow_pattern: Optional[list[str]] = None,
                   ignore_pattern: Optional[list[str]] = None) -> None:
        """
        Pull files from object storage into the temporary directory.

        Args:
            model_path: The object storage path of the model.
            allow_pattern: A list of patterns of which files to pull.
            ignore_pattern: A list of patterns of which files not to pull.

        """
        if not model_path.endswith("/"):
            model_path = model_path + "/"
        runai_pull_files(model_path, self.dir, allow_pattern, ignore_pattern)

dir instance-attribute 

dir = dir_name

__init__ 

__init__(url: str) -> None
Source code in vllm/transformers_utils/runai_utils.py 
61
62
63
64
65
66
67
68
69
70
71
72
73
74
def __init__(self, url: str) -> None:
    if envs.VLLM_ASSETS_CACHE_MODEL_CLEAN:
        for sig in (signal.SIGINT, signal.SIGTERM):
            existing_handler = signal.getsignal(sig)
            signal.signal(sig, self._close_by_signal(existing_handler))

    dir_name = os.path.join(
        get_cache_dir(), "model_streamer",
        hashlib.sha256(str(url).encode()).hexdigest()[:8])
    if os.path.exists(dir_name):
        shutil.rmtree(dir_name)
    os.makedirs(dir_name)
    self.dir = dir_name
    logger.debug("Init object storage, model cache path is: %s", dir_name)

_close 

_close() -> None
Source code in vllm/transformers_utils/runai_utils.py 
76
77
78
def _close(self) -> None:
    if os.path.exists(self.dir):
        shutil.rmtree(self.dir)

_close_by_signal 

_close_by_signal(existing_handler=None)
Source code in vllm/transformers_utils/runai_utils.py 
80
81
82
83
84
85
86
87
def _close_by_signal(self, existing_handler=None):

    def new_handler(signum, frame):
        self._close()
        if existing_handler:
            existing_handler(signum, frame)

    return new_handler

pull_files 

pull_files(
    model_path: str = "",
    allow_pattern: Optional[list[str]] = None,
    ignore_pattern: Optional[list[str]] = None,
) -> None

Pull files from object storage into the temporary directory.

Parameters:

Name Type Description Default
model_path str

The object storage path of the model.

 ''
allow_pattern Optional[list[str]]

A list of patterns of which files to pull.

 None
ignore_pattern Optional[list[str]]

A list of patterns of which files not to pull.

 None
Source code in vllm/transformers_utils/runai_utils.py 
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
def pull_files(self,
               model_path: str = "",
               allow_pattern: Optional[list[str]] = None,
               ignore_pattern: Optional[list[str]] = None) -> None:
    """
    Pull files from object storage into the temporary directory.

    Args:
        model_path: The object storage path of the model.
        allow_pattern: A list of patterns of which files to pull.
        ignore_pattern: A list of patterns of which files not to pull.

    """
    if not model_path.endswith("/"):
        model_path = model_path + "/"
    runai_pull_files(model_path, self.dir, allow_pattern, ignore_pattern)

is_runai_obj_uri 

is_runai_obj_uri(model_or_path: str) -> bool
Source code in vllm/transformers_utils/runai_utils.py 
45
46
def is_runai_obj_uri(model_or_path: str) -> bool:
    return model_or_path.lower().startswith(tuple(SUPPORTED_SCHEMES))

list_safetensors 

list_safetensors(path: str = '') -> list[str]

List full file names from object path and filter by allow pattern.

Parameters:

Name Type Description Default
path str

The object storage path to list from.

 ''

Returns:

Type Description
list[str]

list[str]: List of full object storage paths allowed by the pattern
Source code in vllm/transformers_utils/runai_utils.py 
32
33
34
35
36
37
38
39
40
41
42
def list_safetensors(path: str = "") -> list[str]:
    """
    List full file names from object path and filter by allow pattern.

    Args:
        path: The object storage path to list from.

    Returns:
        list[str]: List of full object storage paths allowed by the pattern
    """
    return runai_list_safetensors(path)