Initial commit

hf3fs_utils/README.md

@@ -0,0 +1,37 @@
+# hf3fs_cli
+
+build:
+```bash
+python3 setup_hf3fs_utils.py bdist_wheel
+```
+
+usage:
+```.bash
+$ hf3fs_cli rmtree --help
+Usage: hf3fs_cli rmtree [OPTIONS] [DIR_PATHS]...
+
+  Move a directory tree to the trash and set an expiration time, it will be automatically deleted after expiration
+
+  Example:
+  hf3fs_cli rmtree <path/to/remove> --expire <expire_time>
+
+  - Use --expire [1h|3h|8h|1d|3d|7d] to specify the expiration time, the directory will be deleted after expiration.
+  - Before expiration, you can restore the directory from the trash using `hf3fs_cli mv <trash_path> <target_path>`.
+  - If you need to free up space immediately, you can use `hf3fs_cli rmtree <trash_path>` to delete the data in the trash immediately, this operation cannot be undone!
+  - Use `ls /path/to/hf3fs/trash` to view the trash.
+
+Options:
+  --expire [1h|3h|8h|1d|3d|7d]  Expiration time, contents in the trash will be automatically deleted after expiration
+  -y, --yes                     Skip confirmation prompt and delete immediately
+  --help                        Show this message and exit.
+
+$ hf3fs_cli mv --help
+Usage: hf3fs_cli mv [OPTIONS] OLD_PATH NEW_PATH
+
+  Move files, supports moving files between different mount points within the same 3FS
+
+Options:
+  --help  Show this message and exit.
+```
+
+If you want to use `rmtree` command, the administrator needs to create a trash directory for each user at `/{3fs_mountpoint}/trash/{user_name}`. The cleanup of the trash directory is handled by the `trash_cleaner`. For instructions on how to use it, please refer to `src/client/trash_cleaner/`.

hf3fs_utils/cli.py

@@ -0,0 +1,192 @@
+import errno
+import click
+import os
+import sys
+import stat
+from typing import Optional, List
+from hf3fs_utils.fs import is_relative_to, FileSystem
+from hf3fs_utils.trash import TRASH_CONFIGS, Trash
+
+MOUNTPOINT = os.environ.get("HF3FS_CLI_MOUNTPOINT", None)
+
+
+def get_filesystem(path: str) -> FileSystem:
+    mountpoint = None
+    if MOUNTPOINT is not None:
+        mountpoint = os.path.abspath(MOUNTPOINT)
+    else:
+        path = os.path.realpath(path)
+        parts = path.split(os.sep)
+        for i in range(1, 4):
+            p = os.sep.join(parts[:i])
+            if os.path.exists(os.path.join(p, "3fs-virt")):
+                mountpoint = p
+            break
+    if not mountpoint:
+        abort(f"{path} is not on 3FS")
+    return FileSystem(mountpoint)
+
+
+def abs_path(path: str) -> str:
+    if ".." in path.split(os.path.sep):
+        abort(f"Path {path} contains '..', which is not supported yet")
+    normpath = os.path.normpath(path)
+    # If the user calls rmtree path/symlink, it should delete the symlink instead of the path it points to
+    # For dir paths, take the realpath, but keep the filename as is
+    dir = os.path.dirname(normpath)
+    filename = os.path.basename(normpath)
+    return os.path.join(os.path.realpath(dir), filename)
+
+
+def abort(msg):
+    click.echo(click.style(msg, fg="red"), err=True)
+    sys.exit(1)
+
+
+@click.group()
+def cli():
+    """
+    3FS command-line tool
+    """
+
+
+@cli.command()
+@click.argument("old_path", type=click.Path(exists=True))
+@click.argument("new_path", type=click.Path())
+def mv(old_path: str, new_path: str):
+    """
+    Move files, supports moving files between different mount points within the same 3FS
+    """
+    try:
+        old_path = abs_path(old_path)
+        new_path = abs_path(new_path)
+
+        try:
+            new_st = os.stat(new_path, follow_symlinks=True)
+            # new_path exists, should be a directory
+            if not stat.S_ISDIR(new_st.st_mode):
+                raise FileExistsError(errno.EEXIST, os.strerror(errno.EEXIST), new_path)
+            # move to new_path/filename
+            new_path = os.path.join(new_path, os.path.basename(old_path))
+        except FileNotFoundError:
+            pass
+
+        fs = get_filesystem(old_path)
+        fs.rename(old_path, new_path)
+        click.echo(f"Move successful: {old_path} -> {new_path}")
+    except AssertionError:
+        raise
+    except Exception as ex:
+        abort(f"Move failed: {ex}")
+
+
+class ExpireType(click.ParamType):
+    def get_metavar(self, param) -> str:
+        return "[1h|3h|8h|1d|3d|7d]"
+
+    def convert(self, value, param, ctx):
+        norm_value = value
+        if norm_value.endswith("hour"):
+            norm_value = norm_value.replace("hour", "h")
+        elif norm_value.endswith("hours"):
+            norm_value = norm_value.replace("hours", "h")
+        elif norm_value.endswith("day"):
+            norm_value = norm_value.replace("day", "d")
+        elif norm_value.endswith("days"):
+            norm_value = norm_value.replace("days", "d")
+
+        if norm_value not in TRASH_CONFIGS.keys():
+            self.fail(f"{value} is invalid, valid options are {self.get_metavar()}", param, ctx)
+        else:
+            return norm_value
+
+
+@cli.command()
+@click.argument("dir_paths", type=click.Path(exists=True), nargs=-1)
+@click.option(
+    "--expire",
+    type=ExpireType(),
+    help="Expiration time, contents in the trash will be automatically deleted after expiration",
+)
+@click.option("-y", "--yes", is_flag=True, default=False, help="Skip confirmation prompt and delete immediately")
+def rmtree(dir_paths: List[str], expire: Optional[str], yes: bool):
+    """
+    Move a directory tree to the trash and set an expiration time, it will be automatically deleted after expiration
+
+    \b
+    Example:
+    hf3fs_cli rmtree <path/to/remove> --expire <expire_time>
+
+    \b
+    - Use --expire [1h|3h|8h|1d|3d|7d] to specify the expiration time, the directory will be deleted after expiration.
+    - Before expiration, you can restore the directory from the trash using `hf3fs_cli mv <trash_path> <target_path>`.
+    - If you need to free up space immediately, you can use `hf3fs_cli rmtree <trash_path>` to delete the data in the trash immediately, this operation cannot be undone!
+    - Use `ls /hf3fs/{cluster}/trash` to view the trash.
+    """
+
+    if not dir_paths:
+        abort(f"Please provide the directory path to delete")
+
+    first_path = abs_path(dir_paths[0])
+    fs = get_filesystem(first_path)
+    fs_trash = Trash(fs)
+
+    clean_trash = is_relative_to(first_path, fs_trash.trash_path)
+    if not clean_trash:
+        if not expire:
+            abort(f"Use --expire [1h|3h|8h|1d|3d|7d] to specify the expiration time")
+    elif expire:
+        abort(f"{first_path} is already in the trash")
+    trash_cfg = TRASH_CONFIGS[expire] if not clean_trash else None
+
+    dir_paths = [abs_path(p) for p in dir_paths]
+    for dir_path in dir_paths:
+        if is_relative_to(dir_path, fs_trash.trash_path) != clean_trash:
+            if clean_trash:
+                abort(f"{dir_path} is not in the trash")
+            else:
+                abort(f"{dir_path} is already in the trash")
+
+    if clean_trash:
+        if len(dir_paths) != 1:
+            msg = (
+                f"Immediately delete the following paths:\n"
+                + "\n".join([f"- {p}" for p in dir_paths])
+                + "\nThis operation cannot be undone"
+            )
+        else:
+            msg = f"Immediately delete {dir_path}, this operation cannot be undone"
+    else:
+        if len(dir_paths) != 1:
+            msg = (
+                f"Move the following paths to the trash:\n"
+                + "\n".join([f"- {p}" for p in dir_paths])
+                + f"\nThey will be automatically deleted after {expire}"
+            )
+        else:
+            msg = f"Move {dir_path} to the trash, it will be automatically deleted after {expire}"
+    if not yes:
+        assert click.confirm(msg, abort=True)
+
+    for dir_path in dir_paths:
+        try:
+            if clean_trash:
+                fs.remove(dir_path, recursive=True)
+                click.echo(f"- Deleted {dir_path}")
+            else:
+                trash_path = fs_trash.move_to_trash(dir_path, trash_cfg)
+                click.echo(f"- Trash path: {trash_path}")
+        except AssertionError:
+            raise
+        except Exception as ex:
+            abort(f"Failed to delete {dir_path}: {ex}")
+
+    if not clean_trash:
+        click.echo(
+            "- Before expiration, you can use 'hf3fs_cli mv <trash_path> <target_path>' to restore, "
+            "or use 'hf3fs_cli rmtree <trash_path>' to delete immediately and free up space"
+        )
+
+
+if __name__ == "__main__":
+    cli()

hf3fs_utils/fs.py

@@ -0,0 +1,234 @@
+import os
+import fcntl
+import errno
+import struct
+import stat
+import sys
+import pathlib
+from typing import Tuple
+
+
+def is_relative_to(path1, path2) -> bool:
+    try:
+        pathlib.PurePath(path1).relative_to(path2)
+        return True
+    except:
+        return False
+
+
+class FileSystem:
+    HF3FS_IOCTL_MAGIC_CMD = 0x80046802
+    HF3FS_IOCTL_MAGIC_NUM = 0x8F3F5FFF
+
+    HF3FS_IOCTL_VERSION_CMD = 0x80046803
+
+    HF3FS_IOCTL_RENAME_CMD = 0x4218680E
+    HF3FS_IOCTL_RENAME_BUFFER_SIZE = 536
+
+    HF3FS_IOCTL_REMOVE_CMD = 0x4110680F
+    HF3FS_IOCTL_REMOVE_BUFFER_SIZE = 272
+
+    def __init__(self, mountpoint: str) -> None:
+        self.mountpoint = os.path.realpath(mountpoint)
+        self.virt_path = os.path.join(self.mountpoint, "3fs-virt")
+
+        # Check if the mount point is a directory
+        if not os.path.exists(self.mountpoint):
+            raise FileNotFoundError(
+                errno.ENOENT, os.strerror(errno.ENOENT), self.mountpoint
+            )
+        if not os.path.isdir(self.mountpoint):
+            raise NotADirectoryError(
+                errno.ENOTDIR, os.strerror(errno.ENOTDIR), self.mountpoint
+            )
+
+        virt_fd = None
+        try:
+            # Check the 3fs-virt directory
+            virt_fd = os.open(self.virt_path, os.O_RDONLY | os.O_DIRECTORY)
+            virt_st = os.fstat(virt_fd)
+            if not stat.S_ISDIR(virt_st.st_mode):
+                raise NotADirectoryError(
+                    errno.ENOTDIR, os.strerror(errno.ENOTDIR), self.virt_path
+                )
+            self.st_dev = virt_st.st_dev
+            # Check the magic number
+            buffer = bytearray(4)
+            try:
+                self._ioctl(virt_fd, FileSystem.HF3FS_IOCTL_MAGIC_CMD, buffer)
+            except OSError:
+                raise RuntimeError(f"{self.mountpoint} is not a 3FS mount point")
+            magic_number = struct.unpack("I", buffer)[0]
+            expected_magic_number = FileSystem.HF3FS_IOCTL_MAGIC_NUM
+            if magic_number != expected_magic_number:
+                raise RuntimeError(
+                    f"{self.mountpoint} is not a 3FS mount point, "
+                    f"magic number {magic_number:x} != {expected_magic_number:x}"
+                )
+            # Check if the required ioctl is supported
+            ioctl_version = -1
+            try:
+                buffer = bytearray(4)
+                self._ioctl(virt_fd, FileSystem.HF3FS_IOCTL_VERSION_CMD, buffer)
+                ioctl_version = int.from_bytes(buffer, sys.byteorder, signed=False)
+            except OSError:
+                pass
+            assert ioctl_version >= 1
+        finally:
+            if virt_fd is not None:
+                os.close(virt_fd)
+
+    def _check_user(self):
+        if os.geteuid() == 0 or os.getegid() == 0:
+            raise RuntimeError(f"root user not allowed")
+
+    def _encode_filename(self, name: str) -> bytes:
+        assert name and os.sep not in name, name
+        name_bytes = name.encode("utf8")
+        if len(name_bytes) > 255:
+            raise OSError(errno.ENAMETOOLONG, os.strerror(errno.ENAMETOOLONG), name)
+        return name_bytes
+
+    def opendir(self, dir_path: str) -> Tuple[int, os.stat_result]:
+        dir_fd = os.open(dir_path, os.O_DIRECTORY | os.O_RDONLY)
+        try:
+            try:
+                dir_st = os.fstat(dir_fd)
+            except OSError as ex:
+                ex.filename = dir_path
+                raise
+
+            if not stat.S_ISDIR(dir_st.st_mode):
+                raise NotADirectoryError(
+                    errno.ENOTDIR, os.strerror(errno.ENOTDIR), dir_path
+                )
+            if dir_st.st_dev != self.st_dev:
+                raise RuntimeError(f"{dir_path} is not under the 3FS mount point {self.mountpoint}")
+            if dir_st.st_ino & 0xF000000000000000:
+                raise RuntimeError(f"{dir_path} is a virtual path")
+
+            return dir_fd, dir_st
+        except:
+            os.close(dir_fd)
+            raise
+
+    def split_path(self, path: str) -> Tuple[int, os.stat_result, str]:
+        filename = os.path.basename(path)
+        if not filename:
+            raise RuntimeError(f"{path} has no filename")
+        if filename in [".", ".."]:
+            raise RuntimeError(f"{path} filename is {filename}")
+        if len(filename.encode("utf8")) > 255:
+            raise OSError(errno.ENAMETOOLONG, os.strerror(errno.ENAMETOOLONG), path)
+
+        dir = os.path.dirname(path) or "."
+        dir_fd, dir_st = self.opendir(dir)
+        return dir_fd, dir_st, filename
+
+    def rename(self, old_path: str, new_path: str) -> None:
+        self._check_user()
+        if is_relative_to(
+            os.path.realpath(new_path), os.path.join(self.mountpoint, "trash")
+        ):
+            raise RuntimeError(f"{new_path} is in the trash")
+
+        old_dir_fd = None
+        new_dir_fd = None
+        try:
+            old_dir_fd, old_dir_st, old_filename = self.split_path(old_path)
+            new_dir_fd, new_dir_st, new_filename = self.split_path(new_path)
+
+            try:
+                old_st = os.stat(old_filename, dir_fd=old_dir_fd, follow_symlinks=False)
+                if stat.S_ISLNK(old_st.st_mode):
+                    raise RuntimeError(f"{old_path} is symlink")
+            except OSError as ex:
+                ex.filename = old_path
+                raise
+
+            try:
+                os.stat(new_filename, dir_fd=new_dir_fd, follow_symlinks=False)
+                raise FileExistsError(errno.EEXIST, os.strerror(errno.EEXIST), new_path)
+            except FileNotFoundError:
+                pass
+            except OSError as ex:
+                ex.filename = new_path
+                raise
+
+            try:
+                self._rename_ioctl(
+                    old_dir_fd,
+                    old_dir_st.st_ino,
+                    old_filename,
+                    new_dir_st.st_ino,
+                    new_filename,
+                    False,
+                )
+            except OSError as ex:
+                ex.filename = old_path
+                ex.filename2 = new_path
+                raise
+        finally:
+            if old_dir_fd is not None:
+                os.close(old_dir_fd)
+            if new_dir_fd is not None:
+                os.close(new_dir_fd)
+
+    def _rename_ioctl(
+        self,
+        old_dir_fd: int,
+        old_dir_ino: int,
+        old_filename: str,
+        new_dir_ino: int,
+        new_filename: str,
+        move_to_trash: bool,
+    ) -> None:
+        assert old_filename and not os.path.sep in old_filename, old_filename
+        assert new_filename and not os.path.sep in new_filename, new_filename
+
+        cmd = FileSystem.HF3FS_IOCTL_RENAME_CMD
+        buffer = struct.pack(
+            "N256sN256s?",
+            old_dir_ino,
+            self._encode_filename(old_filename),
+            new_dir_ino,
+            self._encode_filename(new_filename),
+            move_to_trash,
+        ).ljust(FileSystem.HF3FS_IOCTL_RENAME_BUFFER_SIZE)
+        self._ioctl(old_dir_fd, cmd, buffer)
+
+    def remove(self, path: str, recursive: bool) -> None:
+        dir_fd = None
+        try:
+            dir_fd, dir_st, filename = self.split_path(path)
+            st = os.stat(filename, dir_fd=dir_fd, follow_symlinks=False)
+            if stat.S_ISLNK(st.st_mode):
+                raise RuntimeError(f"{path} is symlink")
+            if stat.S_ISDIR(st.st_mode) and recursive:
+                # The user must be the owner of the directory and have rwx permissions
+                imode = stat.S_IMODE(st.st_mode)
+                if st.st_uid != os.geteuid() or (imode & 0o700) != 0o700:
+                    raise PermissionError(errno.EPERM, os.strerror(errno.EPERM), path)
+
+            try:
+                self._remove_ioctl(dir_fd, dir_st.st_ino, filename, recursive)
+            except OSError as ex:
+                ex.filename = path
+                raise ex
+        finally:
+            if dir_fd is not None:
+                os.close(dir_fd)
+
+    def _remove_ioctl(
+        self, parent_fd: int, parent_ino: int, filename: str, recursive: bool
+    ) -> None:
+        assert filename and os.sep not in filename, filename
+        cmd = FileSystem.HF3FS_IOCTL_REMOVE_CMD
+        buffer = struct.pack(
+            "N256s?", parent_ino, self._encode_filename(filename), recursive
+        ).ljust(FileSystem.HF3FS_IOCTL_REMOVE_BUFFER_SIZE)
+        self._ioctl(parent_fd, cmd, buffer)
+
+    def _ioctl(self, fd: int, cmd: int, buffer):
+        self._check_user()
+        return fcntl.ioctl(fd, cmd, buffer)

hf3fs_utils/hf3fs_cli

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from hf3fs_utils import cli
+
+if __name__ == "__main__":
+    cli.cli(max_content_width=120)

hf3fs_utils/trash.py

@@ -0,0 +1,176 @@
+import os
+import dataclasses
+import pwd
+import stat
+import errno
+import time
+from typing import Optional
+from datetime import datetime, timedelta, timezone
+from . import fs
+
+UTC8_TZ = timezone(timedelta(hours=8))
+DATE_FORMAT = "%Y%m%d_%H%M"
+BASE_TIMESTAMP = int(datetime(year=1980, month=1, day=1, tzinfo=UTC8_TZ).timestamp())
+
+
+def format_date(t: datetime) -> str:
+    assert t.tzinfo
+    return t.astimezone(tz=UTC8_TZ).strftime(DATE_FORMAT)
+
+
+def parse_date(t: str) -> datetime:
+    return datetime.strptime(t, DATE_FORMAT).replace(tzinfo=UTC8_TZ)
+
+
+def get_timestamp_us() -> int:
+    timestamp_seconds = time.time()
+    return int(timestamp_seconds * 1_000_000)
+
+
+@dataclasses.dataclass
+class TrashConfig:
+    name: str
+    expire: timedelta
+    time_slice: timedelta
+
+    def __post_init__(self):
+        assert self.name and "-" not in self.name, f"invalid name {self.name}"
+        assert self.expire >= timedelta(minutes=1), self.expire
+        assert self.time_slice >= timedelta(minutes=1), self.time_slice
+        assert self.time_slice < self.expire, (self.time_slice, self.expire)
+
+    def current_dir(self) -> str:
+        base_timestamp = BASE_TIMESTAMP
+        current_timestamp = int(datetime.now(tz=UTC8_TZ).timestamp())
+        assert current_timestamp > base_timestamp, current_timestamp
+
+        time_slice_seconds = int(self.time_slice.total_seconds())
+        expire_seconds = int(self.expire.total_seconds())
+        assert time_slice_seconds and expire_seconds, repr(self)
+        start_timestamp = (
+            (current_timestamp - base_timestamp) // time_slice_seconds
+        ) * time_slice_seconds + base_timestamp
+        end_timestamp = start_timestamp + expire_seconds + time_slice_seconds
+        start_datetime = datetime.fromtimestamp(start_timestamp, tz=UTC8_TZ)
+        end_datetime = datetime.fromtimestamp(end_timestamp, tz=UTC8_TZ)
+
+        return f"{self.name}-{format_date(start_datetime)}-{format_date(end_datetime)}"
+
+
+TRASH_CONFIGS = {
+    "1h": TrashConfig("1h", timedelta(hours=1), timedelta(minutes=10)),
+    "3h": TrashConfig("3h", timedelta(hours=3), timedelta(minutes=30)),
+    "8h": TrashConfig("8h", timedelta(hours=8), timedelta(minutes=30)),
+    "1d": TrashConfig("1d", timedelta(days=1), timedelta(hours=1)),
+    "3d": TrashConfig("3d", timedelta(days=3), timedelta(days=1)),
+    "7d": TrashConfig("7d", timedelta(days=7), timedelta(days=1)),
+}
+
+
+class Trash:
+    def __init__(
+        self,
+        filesystem: fs.FileSystem,
+        user: Optional[int] = None,
+        user_name: Optional[str] = None,
+    ) -> None:
+        if user is None:
+            user = os.geteuid()
+        assert isinstance(user, int), user
+        if user_name is None:
+            user_name = pwd.getpwuid(user).pw_name
+        if user == 0:
+            raise RuntimeError(f"hf3fs trash does not support root user")
+
+        # Check if the trash directory is mounted
+        trash = os.path.join(filesystem.mountpoint, "trash")
+        if not os.path.exists(trash):
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), trash)
+
+        # Check if the user's trash directory exists
+        user_trash = os.path.join(filesystem.mountpoint, "trash", user_name)
+        if not os.path.exists(user_trash):
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), user_trash)
+
+        user_trash_fd, user_trash_st = filesystem.opendir(user_trash)
+        os.close(user_trash_fd)
+        assert stat.S_ISDIR(user_trash_st.st_mode)
+        if user_trash_st.st_uid != user:
+            raise RuntimeError(
+                f"Trash directory {user_trash}, owner {user_trash_st.st_uid} != {user}"
+            )
+
+        self.filesystem = filesystem
+        self.user = user
+        self.user_name = user_name
+        self.trash_path = trash
+        self.user_trash_path = user_trash
+
+    def _check_user(self):
+        euid = os.geteuid()
+        if euid != self.user:
+            raise RuntimeError(f"euid {euid} != trash owner {self.user}")
+
+    def move_to_trash(
+        self,
+        path: str,
+        config: TrashConfig,
+        trash_name: Optional[str] = None,
+        append_timestamp_if_exists: bool = True,
+    ) -> str:
+        self._check_user()
+        assert isinstance(config, TrashConfig), f"invalid trash config {config}"
+
+        dir_fd = None
+        trash_dir_fd = None
+        try:
+            dir_fd, dir_st, filename = self.filesystem.split_path(path)
+            try:
+                st = os.stat(filename, dir_fd=dir_fd, follow_symlinks=False)
+            except OSError as ex:
+                ex.filename = path
+                raise
+
+            if stat.S_ISDIR(st.st_mode):
+                # The user must be the owner of the directory and have rwx permissions.
+                imode = stat.S_IMODE(st.st_mode)
+                if st.st_uid != os.geteuid() or (imode & 0o700) != 0o700:
+                    raise PermissionError(errno.EPERM, os.strerror(errno.EPERM), path)
+
+            trash_dir = os.path.join(self.user_trash_path, config.current_dir())
+            try:
+                os.mkdir(trash_dir, 0o755)
+            except FileExistsError:
+                pass
+
+            trash_dir_fd, trash_dir_st = self.filesystem.opendir(trash_dir)
+
+            trash_name = trash_name or filename
+            current_trash_name = trash_name
+            retry = 0
+            while True:
+                retry += 1
+                try:
+                    self.filesystem._rename_ioctl(
+                        dir_fd,
+                        dir_st.st_ino,
+                        filename,
+                        trash_dir_st.st_ino,
+                        current_trash_name,
+                        True,
+                    )
+                    return os.path.join(trash_dir, current_trash_name)
+                except OSError as ex:
+                    if (
+                        ex.errno in (errno.ENOTDIR, errno.EEXIST, errno.ENOTEMPTY)
+                        and append_timestamp_if_exists
+                        and retry < 10
+                    ):
+                        current_trash_name = f"{trash_name[0:200]}.{get_timestamp_us()}"
+                    else:
+                        raise
+        finally:
+            if dir_fd is not None:
+                os.close(dir_fd)
+            if trash_dir_fd is not None:
+                os.close(trash_dir_fd)