Skip to content

utils

ZipFileLongPaths

Bases: ZipFile

Allows longer paths in zip files.

Regular DOS paths are limited to MAX_PATH (260) characters, including the string's terminating NUL character. That limit can be exceeded by using an extended-length path that starts with the '\?' prefix.

Source code in client/ayon_third_party/utils.py
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
class ZipFileLongPaths(zipfile.ZipFile):
    """Allows longer paths in zip files.

    Regular DOS paths are limited to MAX_PATH (260) characters, including
    the string's terminating NUL character.
    That limit can be exceeded by using an extended-length path that
    starts with the '\\?\' prefix.
    """
    _is_windows = platform.system().lower() == "windows"

    def _extract_member(self, member, tpath, pwd):
        if self._is_windows:
            tpath = os.path.abspath(tpath)
            if tpath.startswith("\\\\"):
                tpath = "\\\\?\\UNC\\" + tpath[2:]
            else:
                tpath = "\\\\?\\" + tpath

        return super()._extract_member(member, tpath, pwd)

calculate_file_checksum(filepath, checksum_algorithm, chunk_size=10000)

Calculate file checksum for given algorithm.

Parameters:

Name Type Description Default
filepath str

Path to a file.

required
checksum_algorithm str

Algorithm to use. ('md5', 'sha1', 'sha256')

required
chunk_size int

Chunk size to read file. Defaults to 10000.

10000

Returns:

Name Type Description
str str

Calculated checksum.

Raises:

Type Description
ValueError

File not found or unknown checksum algorithm.

Source code in client/ayon_third_party/utils.py
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
def calculate_file_checksum(
    filepath: str,
    checksum_algorithm: str,
    chunk_size: int = 10000,
) -> str:
    """Calculate file checksum for given algorithm.

    Args:
        filepath (str): Path to a file.
        checksum_algorithm (str): Algorithm to use. ('md5', 'sha1', 'sha256')
        chunk_size (int): Chunk size to read file.
            Defaults to 10000.

    Returns:
        str: Calculated checksum.

    Raises:
        ValueError: File not found or unknown checksum algorithm.

    """
    if not filepath:
        raise ValueError("Filepath is empty.")

    if not os.path.exists(filepath):
        raise ValueError(f"{filepath} doesn't exist.")

    if not os.path.isfile(filepath):
        raise ValueError(f"{filepath} is not a file.")

    func = getattr(hashlib, checksum_algorithm, None)
    if func is None:
        raise ValueError(
            f"Unknown checksum algorithm '{checksum_algorithm}'"
        )

    hash_obj = func()
    with open(filepath, "rb") as f:
        for chunk in iter(lambda: f.read(chunk_size), b""):
            hash_obj.update(chunk)
    return hash_obj.hexdigest()

download_ffmpeg(progress=None)

Download ffmpeg from server.

Todos

Add safeguard to avoid downloading of the file from multiple processes at once.

Parameters:

Name Type Description Default
progress TransferProgress

Keep track about download.

None
Source code in client/ayon_third_party/utils.py
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
def download_ffmpeg(
    progress: Optional[TransferProgress] = None,
):
    """Download ffmpeg from server.

    Todos:
        Add safeguard to avoid downloading of the file from multiple
            processes at once.

    Args:
        progress (ayon_api.TransferProgress): Keep track about download.

    """

    files_info = get_server_files_info()
    file_info = _find_file_info("ffmpeg", files_info)
    if file_info is None:
        raise ValueError((
            "Couldn't find ffmpeg source file for platform '{}'"
        ).format(platform.system()))

    dirpath = get_downloaded_ffmpeg_root()
    log.debug(f"Downloading ffmpeg into: '{dirpath}'")
    if not _download_file(file_info, dirpath, progress=progress):
        log.debug("Other processed already downloaded and extracted ffmpeg.")

    _FFmpegArgs.download_needed = False
    _FFmpegArgs.downloaded_root = NOT_SET

extract_archive_file(archive_file, dst_folder=None)

Extract archived file to a directory.

Parameters:

Name Type Description Default
archive_file str

Path to a archive file.

required
dst_folder Optional[str]

Directory where content will be extracted. By default, same folder where archive file is.

None
Source code in client/ayon_third_party/utils.py
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
def extract_archive_file(
    archive_file: str,
    dst_folder: Optional[str] = None,
):
    """Extract archived file to a directory.

    Args:
        archive_file (str): Path to a archive file.
        dst_folder (Optional[str]): Directory where content will be extracted.
            By default, same folder where archive file is.

    """
    if not dst_folder:
        dst_folder = os.path.dirname(archive_file)

    archive_ext, archive_type = get_archive_ext_and_type(archive_file)

    print("Extracting {} -> {}".format(archive_file, dst_folder))
    if archive_type is None:
        _, ext = os.path.splitext(archive_file)
        raise ValueError((
            f"Invalid file extension \"{ext}\"."
            f" Expected {', '.join(IMPLEMENTED_ARCHIVE_FORMATS)}"
        ))

    if archive_type == "zip":
        zip_file = ZipFileLongPaths(archive_file)
        zip_file.extractall(dst_folder)
        zip_file.close()

    elif archive_type == "tar":
        if archive_ext == ".tar":
            tar_type = "r:"
        elif archive_ext.endswith(".xz"):
            tar_type = "r:xz"
        elif archive_ext.endswith(".gz"):
            tar_type = "r:gz"
        elif archive_ext.endswith(".bz2"):
            tar_type = "r:bz2"
        else:
            tar_type = "r:*"

        try:
            tar_file = tarfile.open(archive_file, tar_type)
        except tarfile.ReadError:
            raise ValueError("corrupted archive")

        tar_file.extractall(dst_folder)
        tar_file.close()

get_archive_ext_and_type(archive_file)

Get archive extension and type.

Parameters:

Name Type Description Default
archive_file str

Path to archive file.

required

Returns:

Type Description
Tuple[Optional[str], Optional[str]]

Tuple[str, str]: Archive extension and type.

Source code in client/ayon_third_party/utils.py
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
def get_archive_ext_and_type(
    archive_file: str
) -> Tuple[Optional[str], Optional[str]]:
    """Get archive extension and type.

    Args:
        archive_file (str): Path to archive file.

    Returns:
        Tuple[str, str]: Archive extension and type.

    """
    tmp_name = archive_file.lower()
    if tmp_name.endswith(".zip"):
        return ".zip", "zip"

    for ext in (
        ".tar",
        ".tgz",
        ".tar.gz",
        ".tar.xz",
        ".tar.bz2",
    ):
        if tmp_name.endswith(ext):
            return ext, "tar"

    return None, None

get_ffmpeg_arguments(tool_name='ffmpeg')

Get arguments to run one of ffmpeg tools.

Parameters:

Name Type Description Default
tool_name FFmpegToolname

Name of tool for which arguments should be returned.

'ffmpeg'

Returns:

Type Description
Optional[List[str]]

list[str]: Path to OpenImageIO directory.

Source code in client/ayon_third_party/utils.py
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
def get_ffmpeg_arguments(
    tool_name: "FFmpegToolname" = "ffmpeg"
) -> Optional[List[str]]:
    """Get arguments to run one of ffmpeg tools.

    Args:
        tool_name (FFmpegToolname): Name of
            tool for which arguments should be returned.

    Returns:
        list[str]: Path to OpenImageIO directory.

    """
    args = _FFmpegArgs.tools.get(tool_name, NOT_SET)
    if args is NOT_SET:
        args = _fill_ffmpeg_tool_args(tool_name)
    return copy.deepcopy(args)

get_oiio_arguments(tool_name='oiiotool')

Get arguments to run one of OpenImageIO tools.

Possible OIIO tools

oiiotool, maketx, iv, iinfo, igrep, idiff, iconvert

Parameters:

Name Type Description Default
tool_name OIIOToolName

Name of OIIO tool.

'oiiotool'

Returns:

Name Type Description
str Optional[List[str]]

Path to zip info file.

Source code in client/ayon_third_party/utils.py
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
def get_oiio_arguments(
    tool_name: "OIIOToolName" = "oiiotool"
) -> Optional[List[str]]:
    """Get arguments to run one of OpenImageIO tools.

    Possible OIIO tools:
        oiiotool, maketx, iv, iinfo, igrep, idiff, iconvert

    Args:
        tool_name (OIIOToolName): Name of OIIO tool.

    Returns:
        str: Path to zip info file.

    """
    args = _OIIOArgs.tools.get(tool_name, NOT_SET)
    if args is NOT_SET:
        args = _fill_oiio_tool_args(tool_name)
    return copy.deepcopy(args)

get_server_files_info()

Receive zip file info from server.

Information must contain at least 'filename' and 'hash' with md5 zip file hash.

Returns:

Type Description
List[ToolDownloadInfo]

list[dict[str, str]]: Information about files on server.

Source code in client/ayon_third_party/utils.py
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
def get_server_files_info() -> List["ToolDownloadInfo"]:
    """Receive zip file info from server.

    Information must contain at least 'filename' and 'hash' with md5 zip
    file hash.

    Returns:
        list[dict[str, str]]: Information about files on server.

    """
    # Cache server files info, they won't change
    if _ThirdPartyCache.server_files_info is None:
        endpoint = _get_addon_endpoint()
        response = ayon_api.get(f"{endpoint}/files_info")
        response.raise_for_status()
        _ThirdPartyCache.server_files_info = response.data
    return copy.deepcopy(_ThirdPartyCache.server_files_info)

is_ffmpeg_download_needed(addon_settings=None)

Check if is download needed.

Returns:

Name Type Description
bool bool

Should be config downloaded.

Source code in client/ayon_third_party/utils.py
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
def is_ffmpeg_download_needed(
    addon_settings: Optional[Dict[str, Any]] = None
) -> bool:
    """Check if is download needed.

    Returns:
        bool: Should be config downloaded.

    """
    if _FFmpegArgs.download_needed is not None:
        return _FFmpegArgs.download_needed

    if addon_settings is None:
        addon_settings = get_addon_settings()
    ffmpeg_settings = addon_settings["ffmpeg"]
    download_needed = False
    if ffmpeg_settings["use_downloaded"]:
        # Check what is required by server
        ffmpeg_root = get_downloaded_ffmpeg_root()
        progress_info = {}
        if ffmpeg_root:
            progress_path = os.path.join(
                ffmpeg_root, DIST_PROGRESS_FILENAME
            )
            progress_info = _read_progress_file(progress_path)
        download_needed = progress_info.get("state") != "done"

    _FFmpegArgs.download_needed = download_needed
    return _FFmpegArgs.download_needed

is_oiio_download_needed(addon_settings=None)

Check if is download needed.

Returns:

Name Type Description
bool bool

Should be config downloaded.

Source code in client/ayon_third_party/utils.py
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
def is_oiio_download_needed(
    addon_settings: Optional[Dict[str, Any]] = None
) -> bool:
    """Check if is download needed.

    Returns:
        bool: Should be config downloaded.

    """
    if _OIIOArgs.download_needed is not None:
        return _OIIOArgs.download_needed

    if addon_settings is None:
        addon_settings = get_addon_settings()
    oiio_settings = addon_settings["oiio"]

    download_needed = False
    if oiio_settings["use_downloaded"]:
        oiio_root = get_downloaded_oiio_root()
        progress_info = {}
        if oiio_root:
            progress_path = os.path.join(
                oiio_root, DIST_PROGRESS_FILENAME
            )
            progress_info = _read_progress_file(progress_path)
        download_needed = progress_info.get("state") != "done"
    _OIIOArgs.download_needed = download_needed
    return _OIIOArgs.download_needed

validate_ffmpeg_args(args)

Validate ffmpeg arguments.

Parameters:

Name Type Description Default
args list[str]

ffmpeg arguments.

required

Returns:

Name Type Description
bool bool

True if arguments are valid.

Source code in client/ayon_third_party/utils.py
341
342
343
344
345
346
347
348
349
350
351
352
353
def validate_ffmpeg_args(args: List[str]) -> bool:
    """Validate ffmpeg arguments.

    Args:
        args (list[str]): ffmpeg arguments.

    Returns:
        bool: True if arguments are valid.

    """
    if not args:
        return False
    return _check_args_returncode(args + ["-version"])

validate_file_checksum(filepath, checksum, checksum_algorithm)

Validate file checksum.

Parameters:

Name Type Description Default
filepath str

Path to file.

required
checksum str

Hash of file.

required
checksum_algorithm str

Type of checksum.

required

Returns:

Name Type Description
bool bool

Hash is valid/invalid.

Raises:

Type Description
ValueError

File not found or unknown checksum algorithm.

Source code in client/ayon_third_party/utils.py
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
def validate_file_checksum(
    filepath: str,
    checksum: str,
    checksum_algorithm: str,
) -> bool:
    """Validate file checksum.

    Args:
        filepath (str): Path to file.
        checksum (str): Hash of file.
        checksum_algorithm (str): Type of checksum.

    Returns:
        bool: Hash is valid/invalid.

    Raises:
        ValueError: File not found or unknown checksum algorithm.

    """
    return checksum == calculate_file_checksum(filepath, checksum_algorithm)

validate_oiio_args(args)

Validate oiio arguments.

Parameters:

Name Type Description Default
args list[str]

oiio arguments.

required

Returns:

Name Type Description
bool bool

True if arguments are valid.

Source code in client/ayon_third_party/utils.py
356
357
358
359
360
361
362
363
364
365
366
367
368
def validate_oiio_args(args: List[str]) -> bool:
    """Validate oiio arguments.

    Args:
        args (list[str]): oiio arguments.

    Returns:
        bool: True if arguments are valid.

    """
    if not args:
        return False
    return _check_args_returncode(args + ["--help"])