ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-14 16:46:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix can't write more than 2 GB to a file

Browse Source
This commit is contained in:
allegroai 2022-12-22 21:48:07 +02:00
parent 17dfa2b92f
commit d723299f1e
4 changed files with 10 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
clearml/binding/artifacts.py
View File

@ -562,8 +562,8 @@ class Artifacts(object):
if serialized_text is not None: if serialized_text is not None:
override_filename_in_uri = name + override_filename_ext_in_uri override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + ".", suffix=override_filename_ext_in_uri) fd, local_filename = mkstemp(prefix=quote(name, safe="") + ".", suffix=override_filename_ext_in_uri)
os.write(fd, bytes(serialized_text.encode())) with open(fd, "w") as f:
os.close(fd) f.write(serialized_text)
preview = preview or serialized_text preview = preview or serialized_text
if len(preview) < self.max_preview_size_bytes: if len(preview) < self.max_preview_size_bytes:
artifact_type_data.preview = preview artifact_type_data.preview = preview

11
clearml/binding/frameworks/tensorflow_bind.py
View File

@ -395,17 +395,14 @@ class EventTrainsWriter(object):
suffix=guess_extension(im.get_format_mimetype()) if hasattr(im, 'get_format_mimetype') suffix=guess_extension(im.get_format_mimetype()) if hasattr(im, 'get_format_mimetype')
else ".{}".format(str(im.format).lower()) else ".{}".format(str(im.format).lower())
) )
os.write(fd, imdata) with open(fd, "wb") as f:
os.close(fd) f.write(imdata)
return temp_file return temp_file
image = np.asarray(im)
output.close() output.close()
if height is not None and height > 0 and width is not None and width > 0: if height is not None and height > 0 and width is not None and width > 0:
# noinspection PyArgumentList val = np.array(im).reshape((height, width, -1)).astype(np.uint8)
val = image.reshape(height, width, -1).astype(np.uint8)
else: else:
val = image.astype(np.uint8) val = np.array(im).astype(np.uint8)
if val.ndim == 3 and val.shape[2] == 3: if val.ndim == 3 and val.shape[2] == 3:
if self._visualization_mode == 'BGR': if self._visualization_mode == 'BGR':
val = val[:, :, [2, 1, 0]] val = val[:, :, [2, 1, 0]]

5
clearml/binding/matplotlib_bind.py
View File

@ -1,6 +1,5 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import os
import sys import sys
import threading import threading
from copy import deepcopy from copy import deepcopy
@ -559,8 +558,8 @@ class PatchedMatplotlib:
facecolor=None) facecolor=None)
buffer_.seek(0) buffer_.seek(0)
fd, image = mkstemp(suffix='.' + image_format) fd, image = mkstemp(suffix='.' + image_format)
os.write(fd, buffer_.read()) with open(fd, "wb") as f:
os.close(fd) f.write(buffer_.read())
# check if we need to restore the active object # check if we need to restore the active object
if set_active and not _pylab_helpers.Gcf.get_active() and stored_figure: if set_active and not _pylab_helpers.Gcf.get_active() and stored_figure:

4
clearml/task.py
View File

@ -1633,8 +1633,8 @@ class Task(_Task):
fd, local_filename = mkstemp(prefix='clearml_task_config_', fd, local_filename = mkstemp(prefix='clearml_task_config_',
suffix=configuration_path.suffixes[-1] if suffix=configuration_path.suffixes[-1] if
configuration_path.suffixes else '.txt') configuration_path.suffixes else '.txt')
os.write(fd, configuration_text.encode('utf-8')) with open(fd, "w") as f:
os.close(fd) f.write(configuration_text)
if pathlib_Path: if pathlib_Path:
return pathlib_Path(local_filename) return pathlib_Path(local_filename)
return Path(local_filename) if isinstance(configuration, Path) else local_filename return Path(local_filename) if isinstance(configuration, Path) else local_filename