Skip to content

thumbnails

ThumbnailsCache

Cache of thumbnails on local storage.

Thumbnails are cached to appdirs to predefined directory. Each project has own subfolder with thumbnails -> that's because each project has own thumbnail id validation and file names are thumbnail ids with matching extension. Extensions are predefined (.png and .jpeg).

Cache has cleanup mechanism which is triggered on initialized by default.

The cleanup has 2 levels: 1. soft cleanup which remove all files that are older then 'days_alive' 2. max size cleanup which remove all files until the thumbnails folder contains less then 'max_filesize' - this is time consuming so it's not triggered automatically

Parameters:

Name Type Description Default
cleanup bool

Trigger soft cleanup (Cleanup expired thumbnails).

True
Source code in client/ayon_core/pipeline/thumbnails.py
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
class ThumbnailsCache:
    """Cache of thumbnails on local storage.

    Thumbnails are cached to appdirs to predefined directory. Each project has
    own subfolder with thumbnails -> that's because each project has own
    thumbnail id validation and file names are thumbnail ids with matching
    extension. Extensions are predefined (.png and .jpeg).

    Cache has cleanup mechanism which is triggered on initialized by default.

    The cleanup has 2 levels:
    1. soft cleanup which remove all files that are older then 'days_alive'
    2. max size cleanup which remove all files until the thumbnails folder
        contains less then 'max_filesize'
        - this is time consuming so it's not triggered automatically

    Args:
        cleanup (bool): Trigger soft cleanup (Cleanup expired thumbnails).
    """

    # Lifetime of thumbnails (in seconds)
    # - default 3 days
    days_alive = 3
    # Max size of thumbnail directory (in bytes)
    # - default 2 Gb
    max_filesize = 2 * 1024 * 1024 * 1024

    def __init__(self, cleanup=True):
        self._thumbnails_dir = None
        self._days_alive_secs = self.days_alive * 24 * 60 * 60
        if cleanup:
            self.cleanup()

    def get_thumbnails_dir(self):
        """Root directory where thumbnails are stored.

        Returns:
            str: Path to thumbnails root.
        """

        if self._thumbnails_dir is None:
            self._thumbnails_dir = get_launcher_local_dir("thumbnails")
        return self._thumbnails_dir

    thumbnails_dir = property(get_thumbnails_dir)

    def get_thumbnails_dir_file_info(self):
        """Get information about all files in thumbnails directory.

        Returns:
            List[FileInfo]: List of file information about all files.
        """

        thumbnails_dir = self.thumbnails_dir
        files_info = []
        if not os.path.exists(thumbnails_dir):
            return files_info

        for root, _, filenames in os.walk(thumbnails_dir):
            for filename in filenames:
                path = os.path.join(root, filename)
                files_info.append(FileInfo(
                    path, os.path.getsize(path), os.path.getmtime(path)
                ))
        return files_info

    def get_thumbnails_dir_size(self, files_info=None):
        """Got full size of thumbnail directory.

        Args:
            files_info (List[FileInfo]): Prepared file information about
                files in thumbnail directory.

        Returns:
            int: File size of all files in thumbnail directory.
        """

        if files_info is None:
            files_info = self.get_thumbnails_dir_file_info()

        if not files_info:
            return 0

        return sum(
            file_info.size
            for file_info in files_info
        )

    def cleanup(self, check_max_size=False):
        """Cleanup thumbnails directory.

        Args:
            check_max_size (bool): Also cleanup files to match max size of
                thumbnails directory.
        """

        thumbnails_dir = self.get_thumbnails_dir()
        # Skip if thumbnails dir does not exist yet
        if not os.path.exists(thumbnails_dir):
            return

        self._soft_cleanup(thumbnails_dir)
        if check_max_size:
            self._max_size_cleanup(thumbnails_dir)

    def _soft_cleanup(self, thumbnails_dir):
        current_time = time.time()
        for root, _, filenames in os.walk(thumbnails_dir):
            for filename in filenames:
                path = os.path.join(root, filename)
                modification_time = os.path.getmtime(path)
                if current_time - modification_time > self._days_alive_secs:
                    os.remove(path)

    def _max_size_cleanup(self, thumbnails_dir):
        files_info = self.get_thumbnails_dir_file_info()
        size = self.get_thumbnails_dir_size(files_info)
        if size < self.max_filesize:
            return

        sorted_file_info = collections.deque(
            sorted(files_info, key=lambda item: item.modification_time)
        )
        diff = size - self.max_filesize
        while diff > 0:
            if not sorted_file_info:
                break

            file_info = sorted_file_info.popleft()
            diff -= file_info.size
            os.remove(file_info.path)

    def get_thumbnail_filepath(self, project_name, thumbnail_id):
        """Get thumbnail by thumbnail id.

        Args:
            project_name (str): Name of project.
            thumbnail_id (str): Thumbnail id.

        Returns:
            Union[str, None]: Path to thumbnail image or None if thumbnail
                is not cached yet.
        """

        if not thumbnail_id:
            return None

        for ext in (
            ".png",
            ".jpeg",
        ):
            filepath = os.path.join(
                self.thumbnails_dir, project_name, thumbnail_id + ext
            )
            if os.path.exists(filepath):
                return filepath
        return None

    def get_project_dir(self, project_name):
        """Path to root directory for specific project.

        Args:
            project_name (str): Name of project for which root directory path
                should be returned.

        Returns:
            str: Path to root of project's thumbnails.
        """

        return os.path.join(self.thumbnails_dir, project_name)

    def make_sure_project_dir_exists(self, project_name):
        project_dir = self.get_project_dir(project_name)
        if not os.path.exists(project_dir):
            os.makedirs(project_dir)
        return project_dir

    def store_thumbnail(self, project_name, thumbnail_id, content, mime_type):
        """Store thumbnail to cache folder.

        Args:
            project_name (str): Project where the thumbnail belong to.
            thumbnail_id (str): Thumbnail id.
            content (bytes): Byte content of thumbnail file.
            mime_type (str): Type of content.

        Returns:
            str: Path to cached thumbnail image file.
        """

        if mime_type == "image/png":
            ext = ".png"
        elif mime_type == "image/jpeg":
            ext = ".jpeg"
        else:
            raise ValueError(
                "Unknown mime type for thumbnail \"{}\"".format(mime_type))

        project_dir = self.make_sure_project_dir_exists(project_name)
        thumbnail_path = os.path.join(project_dir, thumbnail_id + ext)
        with open(thumbnail_path, "wb") as stream:
            stream.write(content)

        current_time = time.time()
        os.utime(thumbnail_path, (current_time, current_time))

        return thumbnail_path

cleanup(check_max_size=False)

Cleanup thumbnails directory.

Parameters:

Name Type Description Default
check_max_size bool

Also cleanup files to match max size of thumbnails directory.

False
Source code in client/ayon_core/pipeline/thumbnails.py
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
def cleanup(self, check_max_size=False):
    """Cleanup thumbnails directory.

    Args:
        check_max_size (bool): Also cleanup files to match max size of
            thumbnails directory.
    """

    thumbnails_dir = self.get_thumbnails_dir()
    # Skip if thumbnails dir does not exist yet
    if not os.path.exists(thumbnails_dir):
        return

    self._soft_cleanup(thumbnails_dir)
    if check_max_size:
        self._max_size_cleanup(thumbnails_dir)

get_project_dir(project_name)

Path to root directory for specific project.

Parameters:

Name Type Description Default
project_name str

Name of project for which root directory path should be returned.

required

Returns:

Name Type Description
str

Path to root of project's thumbnails.

Source code in client/ayon_core/pipeline/thumbnails.py
174
175
176
177
178
179
180
181
182
183
184
185
def get_project_dir(self, project_name):
    """Path to root directory for specific project.

    Args:
        project_name (str): Name of project for which root directory path
            should be returned.

    Returns:
        str: Path to root of project's thumbnails.
    """

    return os.path.join(self.thumbnails_dir, project_name)

get_thumbnail_filepath(project_name, thumbnail_id)

Get thumbnail by thumbnail id.

Parameters:

Name Type Description Default
project_name str

Name of project.

required
thumbnail_id str

Thumbnail id.

required

Returns:

Type Description

Union[str, None]: Path to thumbnail image or None if thumbnail is not cached yet.

Source code in client/ayon_core/pipeline/thumbnails.py
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
def get_thumbnail_filepath(self, project_name, thumbnail_id):
    """Get thumbnail by thumbnail id.

    Args:
        project_name (str): Name of project.
        thumbnail_id (str): Thumbnail id.

    Returns:
        Union[str, None]: Path to thumbnail image or None if thumbnail
            is not cached yet.
    """

    if not thumbnail_id:
        return None

    for ext in (
        ".png",
        ".jpeg",
    ):
        filepath = os.path.join(
            self.thumbnails_dir, project_name, thumbnail_id + ext
        )
        if os.path.exists(filepath):
            return filepath
    return None

get_thumbnails_dir()

Root directory where thumbnails are stored.

Returns:

Name Type Description
str

Path to thumbnails root.

Source code in client/ayon_core/pipeline/thumbnails.py
49
50
51
52
53
54
55
56
57
58
def get_thumbnails_dir(self):
    """Root directory where thumbnails are stored.

    Returns:
        str: Path to thumbnails root.
    """

    if self._thumbnails_dir is None:
        self._thumbnails_dir = get_launcher_local_dir("thumbnails")
    return self._thumbnails_dir

get_thumbnails_dir_file_info()

Get information about all files in thumbnails directory.

Returns:

Type Description

List[FileInfo]: List of file information about all files.

Source code in client/ayon_core/pipeline/thumbnails.py
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
def get_thumbnails_dir_file_info(self):
    """Get information about all files in thumbnails directory.

    Returns:
        List[FileInfo]: List of file information about all files.
    """

    thumbnails_dir = self.thumbnails_dir
    files_info = []
    if not os.path.exists(thumbnails_dir):
        return files_info

    for root, _, filenames in os.walk(thumbnails_dir):
        for filename in filenames:
            path = os.path.join(root, filename)
            files_info.append(FileInfo(
                path, os.path.getsize(path), os.path.getmtime(path)
            ))
    return files_info

get_thumbnails_dir_size(files_info=None)

Got full size of thumbnail directory.

Parameters:

Name Type Description Default
files_info List[FileInfo]

Prepared file information about files in thumbnail directory.

None

Returns:

Name Type Description
int

File size of all files in thumbnail directory.

Source code in client/ayon_core/pipeline/thumbnails.py
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
def get_thumbnails_dir_size(self, files_info=None):
    """Got full size of thumbnail directory.

    Args:
        files_info (List[FileInfo]): Prepared file information about
            files in thumbnail directory.

    Returns:
        int: File size of all files in thumbnail directory.
    """

    if files_info is None:
        files_info = self.get_thumbnails_dir_file_info()

    if not files_info:
        return 0

    return sum(
        file_info.size
        for file_info in files_info
    )

store_thumbnail(project_name, thumbnail_id, content, mime_type)

Store thumbnail to cache folder.

Parameters:

Name Type Description Default
project_name str

Project where the thumbnail belong to.

required
thumbnail_id str

Thumbnail id.

required
content bytes

Byte content of thumbnail file.

required
mime_type str

Type of content.

required

Returns:

Name Type Description
str

Path to cached thumbnail image file.

Source code in client/ayon_core/pipeline/thumbnails.py
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
def store_thumbnail(self, project_name, thumbnail_id, content, mime_type):
    """Store thumbnail to cache folder.

    Args:
        project_name (str): Project where the thumbnail belong to.
        thumbnail_id (str): Thumbnail id.
        content (bytes): Byte content of thumbnail file.
        mime_type (str): Type of content.

    Returns:
        str: Path to cached thumbnail image file.
    """

    if mime_type == "image/png":
        ext = ".png"
    elif mime_type == "image/jpeg":
        ext = ".jpeg"
    else:
        raise ValueError(
            "Unknown mime type for thumbnail \"{}\"".format(mime_type))

    project_dir = self.make_sure_project_dir_exists(project_name)
    thumbnail_path = os.path.join(project_dir, thumbnail_id + ext)
    with open(thumbnail_path, "wb") as stream:
        stream.write(content)

    current_time = time.time()
    os.utime(thumbnail_path, (current_time, current_time))

    return thumbnail_path

get_thumbnail_path(project_name, thumbnail_id)

Get path to thumbnail image.

Parameters:

Name Type Description Default
project_name str

Project where thumbnail belongs to.

required
thumbnail_id Union[str, None]

Thumbnail id.

required

Returns:

Type Description

Union[str, None]: Path to thumbnail image or None if thumbnail id is not valid or thumbnail was not possible to receive.

Source code in client/ayon_core/pipeline/thumbnails.py
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
def get_thumbnail_path(project_name, thumbnail_id):
    """Get path to thumbnail image.

    Args:
        project_name (str): Project where thumbnail belongs to.
        thumbnail_id (Union[str, None]): Thumbnail id.

    Returns:
        Union[str, None]: Path to thumbnail image or None if thumbnail
            id is not valid or thumbnail was not possible to receive.

    """
    if not thumbnail_id:
        return None

    filepath = _CacheItems.thumbnails_cache.get_thumbnail_filepath(
        project_name, thumbnail_id
    )
    if filepath is not None:
        return filepath

    # 'ayon_api' had a bug, public function
    #   'get_thumbnail_by_id' did not return output of
    #   'ServerAPI' method.
    con = ayon_api.get_server_api_connection()
    result = con.get_thumbnail_by_id(project_name, thumbnail_id)

    if result is not None and result.is_valid:
        return _CacheItems.thumbnails_cache.store_thumbnail(
            project_name,
            thumbnail_id,
            result.content,
            result.content_type
        )
    return None