borgbackup/borg/platform_linux.pyx

import os
import re
import resource
import stat

from .helpers import posix_acl_use_stored_uid_gid, user2uid, group2gid, safe_decode, safe_encode
from .platform_base import SyncFile as BaseSyncFile
from .platform_posix import swidth

from libc cimport errno

API_VERSION = 3

cdef extern from "sys/types.h":
    int ACL_TYPE_ACCESS
    int ACL_TYPE_DEFAULT
    ctypedef off64_t

cdef extern from "sys/acl.h":
    ctypedef struct _acl_t:
        pass
    ctypedef _acl_t *acl_t

    int acl_free(void *obj)
    acl_t acl_get_file(const char *path, int type)
    acl_t acl_set_file(const char *path, int type, acl_t acl)
    acl_t acl_from_text(const char *buf)
    char *acl_to_text(acl_t acl, ssize_t *len)

cdef extern from "acl/libacl.h":
    int acl_extended_file(const char *path)

cdef extern from "fcntl.h":
    int sync_file_range(int fd, off64_t offset, off64_t nbytes, unsigned int flags)
    unsigned int SYNC_FILE_RANGE_WRITE
    unsigned int SYNC_FILE_RANGE_WAIT_BEFORE
    unsigned int SYNC_FILE_RANGE_WAIT_AFTER

cdef extern from "linux/fs.h":
    # ioctls
    int FS_IOC_SETFLAGS
    int FS_IOC_GETFLAGS

    # inode flags
    int FS_NODUMP_FL
    int FS_IMMUTABLE_FL
    int FS_APPEND_FL
    int FS_COMPR_FL

cdef extern from "stropts.h":
    int ioctl(int fildes, int request, ...)

cdef extern from "errno.h":
    int errno

cdef extern from "string.h":
    char *strerror(int errnum)

_comment_re = re.compile(' *#.*', re.M)


BSD_TO_LINUX_FLAGS = {
    stat.UF_NODUMP: FS_NODUMP_FL,
    stat.UF_IMMUTABLE: FS_IMMUTABLE_FL,
    stat.UF_APPEND: FS_APPEND_FL,
    stat.UF_COMPRESSED: FS_COMPR_FL,
}


def set_flags(path, bsd_flags, fd=None):
    if fd is None and stat.S_ISLNK(os.lstat(path).st_mode):
        return
    cdef int flags = 0
    for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items():
        if bsd_flags & bsd_flag:
            flags |= linux_flag
    open_fd = fd is None
    if open_fd:
        fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW)
    try:
        if ioctl(fd, FS_IOC_SETFLAGS, &flags) == -1:
            raise OSError(errno, strerror(errno).decode(), path)
    finally:
        if open_fd:
            os.close(fd)


def get_flags(path, st):
    if stat.S_ISLNK(st.st_mode):
        return 0
    cdef int linux_flags
    fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW)
    try:
        if ioctl(fd, FS_IOC_GETFLAGS, &linux_flags) == -1:
            return 0
    finally:
        os.close(fd)
    bsd_flags = 0
    for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items():
        if linux_flags & linux_flag:
            bsd_flags |= bsd_flag
    return bsd_flags


def acl_use_local_uid_gid(acl):
    """Replace the user/group field with the local uid/gid if possible
    """
    entries = []
    for entry in safe_decode(acl).split('\n'):
        if entry:
            fields = entry.split(':')
            if fields[0] == 'user' and fields[1]:
                fields[1] = str(user2uid(fields[1], fields[3]))
            elif fields[0] == 'group' and fields[1]:
                fields[1] = str(group2gid(fields[1], fields[3]))
            entries.append(':'.join(fields[:3]))
    return safe_encode('\n'.join(entries))


cdef acl_append_numeric_ids(acl):
    """Extend the "POSIX 1003.1e draft standard 17" format with an additional uid/gid field
    """
    entries = []
    for entry in _comment_re.sub('', safe_decode(acl)).split('\n'):
        if entry:
            type, name, permission = entry.split(':')
            if name and type == 'user':
                entries.append(':'.join([type, name, permission, str(user2uid(name, name))]))
            elif name and type == 'group':
                entries.append(':'.join([type, name, permission, str(group2gid(name, name))]))
            else:
                entries.append(entry)
    return safe_encode('\n'.join(entries))


cdef acl_numeric_ids(acl):
    """Replace the "POSIX 1003.1e draft standard 17" user/group field with uid/gid
    """
    entries = []
    for entry in _comment_re.sub('', safe_decode(acl)).split('\n'):
        if entry:
            type, name, permission = entry.split(':')
            if name and type == 'user':
                uid = str(user2uid(name, name))
                entries.append(':'.join([type, uid, permission, uid]))
            elif name and type == 'group':
                gid = str(group2gid(name, name))
                entries.append(':'.join([type, gid, permission, gid]))
            else:
                entries.append(entry)
    return safe_encode('\n'.join(entries))


def acl_get(path, item, st, numeric_owner=False):
    cdef acl_t default_acl = NULL
    cdef acl_t access_acl = NULL
    cdef char *default_text = NULL
    cdef char *access_text = NULL

    p = <bytes>os.fsencode(path)
    if stat.S_ISLNK(st.st_mode) or acl_extended_file(p) <= 0:
        return
    if numeric_owner:
        converter = acl_numeric_ids
    else:
        converter = acl_append_numeric_ids
    try:
        access_acl = acl_get_file(p, ACL_TYPE_ACCESS)
        if access_acl:
            access_text = acl_to_text(access_acl, NULL)
            if access_text:
                item[b'acl_access'] = converter(access_text)
        default_acl = acl_get_file(p, ACL_TYPE_DEFAULT)
        if default_acl:
            default_text = acl_to_text(default_acl, NULL)
            if default_text:
                item[b'acl_default'] = converter(default_text)
    finally:
        acl_free(default_text)
        acl_free(default_acl)
        acl_free(access_text)
        acl_free(access_acl)


def acl_set(path, item, numeric_owner=False):
    cdef acl_t access_acl = NULL
    cdef acl_t default_acl = NULL

    p = <bytes>os.fsencode(path)
    if numeric_owner:
        converter = posix_acl_use_stored_uid_gid
    else:
        converter = acl_use_local_uid_gid
    access_text = item.get(b'acl_access')
    default_text = item.get(b'acl_default')
    if access_text:
        try:
            access_acl = acl_from_text(<bytes>converter(access_text))
            if access_acl:
                acl_set_file(p, ACL_TYPE_ACCESS, access_acl)
        finally:
            acl_free(access_acl)
    if default_text:
        try:
            default_acl = acl_from_text(<bytes>converter(default_text))
            if default_acl:
                acl_set_file(p, ACL_TYPE_DEFAULT, default_acl)
        finally:
            acl_free(default_acl)

cdef _sync_file_range(fd, offset, length, flags):
    assert offset & PAGE_MASK == 0, "offset %d not page-aligned" % offset
    assert length & PAGE_MASK == 0, "length %d not page-aligned" % length
    if sync_file_range(fd, offset, length, flags) != 0:
        raise OSError(errno, os.strerror(errno))
    os.posix_fadvise(fd, offset, length, os.POSIX_FADV_DONTNEED)

cdef unsigned PAGE_MASK = resource.getpagesize() - 1


class SyncFile(BaseSyncFile):
    """
    Implemented using sync_file_range for asynchronous write-out and fdatasync for actual durability.

    "write-out" means that dirty pages (= data that was written) are submitted to an I/O queue and will be send to
    disk in the immediate future.
    """

    def __init__(self, path):
        super().__init__(path)
        self.offset = 0
        self.write_window = (16 * 1024 ** 2) & ~PAGE_MASK
        self.last_sync = 0
        self.pending_sync = None

    def write(self, data):
        self.offset += self.fd.write(data)
        offset = self.offset & ~PAGE_MASK
        if offset >= self.last_sync + self.write_window:
            self.fd.flush()
            _sync_file_range(self.fileno, self.last_sync, offset - self.last_sync, SYNC_FILE_RANGE_WRITE)
            if self.pending_sync is not None:
                _sync_file_range(self.fileno, self.pending_sync, self.last_sync - self.pending_sync,
                                 SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WAIT_AFTER)
            self.pending_sync = self.last_sync
            self.last_sync = offset

    def sync(self):
        self.fd.flush()
        os.fdatasync(self.fileno)
        os.posix_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED)