Attachment 18484 Details for Bug 6741 – rsync stdout filter

rsync stdout filter

postrsync.c (text/plain), 18.32 KB, created by Marc Aurèle La France on 2024-10-25 15:17:14 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Marc Aurèle La France

Created: 2024-10-25 15:17:14 UTC

Size: 18.32 KB

patch

obsolete

>/*
> * Copyright 2022-2024 by Marc AurÃ¨le La France, tsi@tuyoix.net
> *
> * Permission to use, copy, modify, distribute, and sell this software and its
> * documentation for any purpose is hereby granted without fee, provided that
> * the above copyright notice appear in all copies and that both that copyright
> * notice and this permission notice appear in supporting documentation, and
> * that the name of Marc AurÃ¨le La France not be used in advertising or
> * publicity pertaining to distribution of the software without specific,
> * written prior permission.  Marc AurÃ¨le La France makes no representations
> * about the suitability of this software for any purpose.  It is provided
> * "as-is" without express or implied warranty.
> *
> * MARC AURÃLE LA FRANCE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
> * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.  IN NO
> * EVENT SHALL MARC AURÃLE LA FRANCE BE LIABLE FOR ANY SPECIAL, INDIRECT OR
> * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
> * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
> * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
> * OF THIS SOFTWARE.
> */
>
>/* $TSI$ */
>
>/*
> * Post-process `rsync --out-format='-%i %n%L'` stdout to report file
> * transactions in ascending pathname order, except that directory deletions
> * (or replacements) are reported after the deletion of their contents.  This
> * also strips out interim progress lines given deletions may be reported among
> * them.  The file counts at the end of progress lines are ignored as they no
> * longer make sense after reordering transactions.
> *
> * This does not handle total transfer progress lines (i.e. --info=progress2,
> * specified or implied), nor interim progress lines generated by sending a
> * signal to rsync.  Probably other things too.
> *
> * The hyphen in front of the out-format makes locating %i output simpler and
> * more reliable.
> *
> * This should be statically linked to maximise heap space.  Out-of-memory
> * handling here almost always results in slightly incorrect ordering instead
> * of giving up.
> *
> * This is a reaction to https://bugzilla.samba.org/show_bug.cgi?id=6741 which
> * will never be addressed in rsync itself and probably shouldn't be.
> */
>
>/*
> * Revision history (from rsync's perspective):
> * - Clone stripcr, a filter that omits interim progress lines.
> * - Reorder deletions.
> * - Handle escaping of unusual filenames.
> * - Compensate for sort of directory contents that places non-directories
> *   ahead of subdirectories.
> * - Improve hardlink and symlink handling.
> * - Report deletion of directory contents before the directory itself, even if
> *   the latter is replaced with another file type.
> */
>
>/*
> * Known issues:
> * - Non-deletion of hardlink names containing the " => " string.
> * - Non-deletion of symlink names or target names containing the " -> "
> *   string.
> */
>
>#undef  _GNU_SOURCE
>#define _GNU_SOURCE 1   /* For reallocarray() & mempcpy() */
>
>#include <limits.h>
>#include <signal.h>
>#include <stdbool.h>
>#include <stdio.h>
>#include <stdlib.h>
>#include <string.h>
>
>#undef  DELETING
>#define DELETING "-*deleting   "
>
>#undef  FN_OFF
>#define FN_OFF 13       /* Where %n starts in the out-format */
>
>#undef  MASK
>#define MASK ((sizeof(long double) << 1) - 1)   /* malloc() alignment */
>
>#undef  UNAVAILABLE
>#define UNAVAILABLE ((((size_t)(-1LL)) / sizeof(transaction_t)) + 1)
>
>#undef  UNLIMITED
>#define UNLIMITED ((const char *)(-1LL))
>
>/*
> * Until gcc quits whining about casts and assignments to void * pointers that
> * discard qualifiers.  If it ever does...  After all, void data cannot be
> * modified, are therefore implicitly const and no other qualifiers matter.
> */
>#ifndef voidconst
>#define voidconst /* const */
>#endif
>
>typedef struct
>{
>    voidconst char *encoded;
>    voidconst char *decoded;
>    double rate;
>    unsigned int hours;
>    unsigned char minutes;
>    unsigned char seconds;
>    char notime;
>    /* Enough for 64-bit values in decimal with commas and trailing NUL */
>    char filesize[27];
>    char percent;       /* Deletions are more than 100% */
>    char scale;
>} transaction_t;
>
>static transaction_t *Transactions;     /* Maintained in descending order */
>static size_t iTransaction;
>static size_t nTransaction, nTransactions;
>static size_t sTransaction = UNAVAILABLE;
>static voidconst char *Transaction, *transaction;
>
>static const char *line, *p, *prefix = "";
>
>static char *buffer;
>static size_t start, len;
>
>static char percent;
>static bool decoded;
>
>static size_t notices;
>
>static void report(void)
>{
>    printf("\n%zu notice%s issued!\n", notices, (notices == 1) ? "" : "s");
>}
>
>/* No sense in second-guessing malloc() internals.  So-o-o-o... */
>static void die(void)
>{
>    report();
>    puts("\n!!Premature termination.\n");
>    fflush(stdout);             /* Just in case */
>    /* Take down rsync as well */
>    kill(0, SIGTERM);           /* No return ... */
>    exit(SIGTERM + 128);        /* ... supposedly ... */
>}
>
>static void printdeletion(const char *pathname)
>{
>    fputs(DELETING, stdout);
>    fputs(pathname, stdout);
>}
>
>static void printtransaction(void)
>{
>    voidconst char *Encoded = Transactions[nTransaction].encoded;
>
>    if (*prefix != '\0')
>    {
>        fputs(prefix, stdout);
>        prefix = "";
>    }
>
>    if (Transactions[nTransaction].percent > 100)
>    {
>        if (Encoded != Transactions[nTransaction].decoded)
>            free(Transactions[nTransaction].decoded);
>
>        printdeletion(Encoded);
>    }
>    else
>    {
>        if (sTransaction == nTransaction)
>            sTransaction = UNAVAILABLE;
>
>        if ((Encoded + FN_OFF) != Transactions[nTransaction].decoded)
>            free(Transactions[nTransaction].decoded);
>
>        fputs(Encoded, stdout);
>
>        if (Transactions[nTransaction].percent == 100)
>        {
>            printf("%15s 100%% %7.2f%cB/s ",
>                   Transactions[nTransaction].filesize,
>                   Transactions[nTransaction].rate,
>                   Transactions[nTransaction].scale);
>
>            if (Transactions[nTransaction].notime == '?')
>                puts("  ??:??:??");
>            else
>                printf("%4u:%02hhu:%02hhu\n",
>                       Transactions[nTransaction].hours,
>                       Transactions[nTransaction].minutes,
>                       Transactions[nTransaction].seconds);
>        }
>    }
>
>    free(Encoded);
>}
>
>static void flushtransactions(void)
>{
>    while (nTransaction > 0)
>    {
>        nTransaction--;
>        printtransaction();
>    }
>}
>
>static bool outofmemory(size_t size)
>{
>    transaction_t *newTransactions;
>
>    /* Last chance before creating more chaos */
>    if (((nTransactions - nTransaction) * sizeof(transaction_t)) >=
>        (size + (MASK + 1 + sizeof(transaction_t))))
>    {
>        Transactions =
>            reallocarray(Transactions, nTransaction + 1, sizeof(transaction_t));
>        nTransactions = nTransaction + 1;
>        prefix = "";
>        return false;
>    }
>
>    if (nTransaction > 0)
>    {
>        nTransaction--;
>        printtransaction();
>        return false;
>    }
>
>    prefix = "";
>
>    if (nTransactions > 1)
>    {
>        Transactions = realloc(Transactions, sizeof(transaction_t));
>        nTransactions = 1;
>        return false;
>    }
>
>    if ((newTransactions = malloc(sizeof(transaction_t))) == NULL)
>        return true;
>
>    if ((nTransactions == 0) || (newTransactions < Transactions))
>    {
>        free(Transactions);
>        Transactions = newTransactions;
>        nTransactions = 1;
>        return false;
>    }
>
>    free(newTransactions);
>    return true;
>}
>
>static void inserttransaction(void)
>{
>    if (nTransaction >= nTransactions)
>    {
>        transaction_t *saveTransactions = Transactions;
>
>        if ((Transactions =
>                reallocarray(Transactions, ++nTransactions,
>                             sizeof(transaction_t))) == NULL)
>        {
>            if (--nTransactions == 0)
>                die();
>            notices++;
>            prefix = "!I";
>            Transactions = saveTransactions;
>            nTransaction--;
>            printtransaction();
>        }
>    }
>
>    /* Binary search candidate */
>    for (iTransaction = nTransaction;  iTransaction-- > 0;  )
>    {
>        int res =
>            strncmp(transaction, Transactions[iTransaction].decoded, len);
>
>        if ((res < 0) ||
>            ((res == 0) && (p == NULL) &&       /* Non-deletion */
>             (Transactions[iTransaction].decoded[len] != '/')))
>            break;
>
>        Transactions[iTransaction + 1] = Transactions[iTransaction];
>    }
>
>    iTransaction++;
>
>    Transactions[iTransaction].encoded = Transaction;
>    Transactions[iTransaction].decoded = transaction;
>    nTransaction++;
>}
>
>static voidconst char *decode(const char *enil, voidconst char *pathname)
>{
>    const char *src, *limit = UNLIMITED;
>    char *dst, *filename;
>
>    decoded = true;
>
>    if (enil[1] == 'h') /* Hard link */
>    {
>        if (((limit = strstr(pathname, " => ")) == NULL) ||
>            (strstr(limit + 4, " => ") != NULL))
>        {
>            limit = UNLIMITED;
>            notices++;
>            printf("!line=%s", enil);
>        }
>    }
>    else
>    if (enil[2] == 'L') /* Symbolic link */
>    {
>        if (((limit = strstr(pathname, " -> ")) == NULL) ||
>            (strstr(limit + 4, " -> ") != NULL))
>        {
>            limit = UNLIMITED;
>            notices++;
>            printf("!line=%s", enil);
>        }
>    }
>
>    for (src = pathname;  (src = strstr(src, "\\#")) != NULL;  src += 2)
>    {
>        if (src >= limit)
>            break;
>
>        if (((src[2] & 0xfc) != '0') ||
>            ((src[3] & 0xf8) != '0') ||
>            ((src[4] & 0xf8) != '0'))
>            continue;
>
>        len = strlen(pathname + ((5 - 1) - 1));
>        while ((filename = malloc(len)) == NULL)
>        {
>            notices++;
>            prefix = "!P";
>            if (outofmemory(len))
>            {
>                decoded = false;
>                return pathname;
>            }
>        }
>
>        dst = mempcpy(filename, pathname, src - pathname);
>
>        while (true)
>        {
>            *dst++ = (src[2] << 6) | ((src[3] & 7) << 3) | (src[4] & 7);
>            src += 5;
>
>            while ((src[0] != '\\') ||
>                   (src[1] != '#') ||
>                   ((src[2] & 0xfc) != '0') ||
>                   ((src[3] & 0xf8) != '0') ||
>                   ((src[4] & 0xf8) != '0'))
>            {
>                if ((*dst++ = *src++) != '\0')
>                {
>                    if (src < limit)
>                        continue;
>
>                    *dst++ = '\n';
>                    *dst++ = '\0';
>                }
>
>                len = (len & ~MASK) + !!(len & MASK);
>
>                if ((filename + len) <= dst)
>                    return filename;
>
>                return realloc(filename, dst - filename);
>            }
>        }
>    }
>
>    if (limit == UNLIMITED)
>        return pathname;
>
>    len = limit - pathname;
>    while ((filename = malloc(len + 2)) == NULL)
>    {
>        notices++;
>        prefix = "!Q";
>        if (outofmemory(len + 2))
>        {
>            decoded = false;
>            return pathname;
>        }
>    }
>
>    dst = mempcpy(filename, pathname, len);
>    *dst++ = '\n';
>    *dst++ = '\0';
>
>    return filename;
>}
>
>static bool isdeletion(void)
>{
>    transaction_t temp;
>
>    if ((p = strstr(line, DELETING)) == NULL)
>        return false;
>
>    if (p == line)
>        return true;
>
>    /* Validate the single interim progress line that precedes this deletion */
>    if (sscanf(line, " %26[0-9,.kKMGTPE]%hhd%%%lf%cB/s%*[ ]%zn%c",
>               temp.filesize,
>               &temp.percent,
>               &temp.rate,
>               &temp.scale,
>               &len,
>               &temp.notime) == 5)
>    {
>        if (temp.notime == '?')
>        {
>            if (!strncmp(line + len, "??:??:??  " DELETING,
>                         10 + strlen(DELETING)))
>                return true;
>        }
>        else
>        {
>            if ((sscanf(line + len, "%u:%2hhu:%2hhu%*[ ]" DELETING "%c",
>                        &temp.hours,
>                        &temp.minutes,
>                        &temp.seconds,
>                        &temp.notime) == 4) &&
>                (temp.minutes < 60) &&
>                (temp.seconds < 60))
>                return true;
>        }
>    }
>
>    p = NULL;
>    return false;
>}
>
>static bool printabletransaction(void)
>{
>    if (nTransaction == 0)
>        return false;
>
>    if (line[2] == 'd') /* Directory */
>        return true;
>
>    if (sTransaction >= UNAVAILABLE)
>        return false;
>
>    if (strcmp(transaction, Transactions[sTransaction].decoded) <= 0)
>        return true;
>
>    return false;
>}
>
>static bool invalidprogress(void)
>{
>    if (Transactions[sTransaction].notime != '?')
>    {
>        size_t length;
>
>        if (sscanf(line + len, "%u:%2hhu:%2hhu%zn",
>                   &Transactions[sTransaction].hours,
>                   &Transactions[sTransaction].minutes,
>                   &Transactions[sTransaction].seconds,
>                   &length) != 3)
>            return true;
>
>        length += len;
>        while (line[length] == ' ')
>            length++;
>
>        if ((line[length] != '\n') && (line[length] != '('))
>            return true;
>
>        if ((Transactions[sTransaction].minutes >= 60) ||
>            (Transactions[sTransaction].seconds >= 60))
>            return true;
>    }
>
>    Transactions[sTransaction].percent = percent;
>
>    return false;
>}
>
>static void print(size_t end)
>{
>    char savec = buffer[end];
>
>    buffer[end] = '\0';
>    line = buffer + start;
>
>    /*
>     * Deletions may insert themselves ahead of other transfers in the same
>     * directory or between iterations of progress lines.  Buffer them so they
>     * can be reinserted in the desired order.
>     */
>    if (isdeletion())
>    {
>        while ((Transaction = strdup(p + FN_OFF)) == NULL)
>        {
>            notices++;
>            prefix = "!D";
>            if (outofmemory(strlen(p + (FN_OFF - 1))))
>            {
>                fputs("!d", stdout);
>                printdeletion(p + FN_OFF);
>                goto done;
>            }
>        }
>
>        transaction = decode(p, Transaction);
>        len = strlen(transaction);
>
>        if ((len > 2) && (transaction[len - 2] == '/'))
>            len--;      /* Directory;  don't compare last newline */
>
>        inserttransaction();
>
>        Transactions[iTransaction].percent = SCHAR_MAX;
>
>        if (iTransaction <= sTransaction)
>            sTransaction++;
>    }
>    else
>    if (*line == '-')   /* -%i */
>    {
>        transaction = decode(line, line + FN_OFF);
>
>        if (printabletransaction())
>        {
>            while (nTransaction-- > 0)
>            {
>                if (strcmp(transaction, Transactions[nTransaction].decoded) < 0)
>                    break;
>
>                printtransaction();
>            }
>
>            nTransaction++;
>        }
>
>        while ((Transaction = strdup(line)) == NULL)
>        {
>            notices++;
>            prefix = "!N";
>            if (outofmemory(strlen(line) + 1))
>                die();
>        }
>
>        if (transaction == (line + FN_OFF))
>        {
>            if (decoded)
>                transaction = Transaction + FN_OFF;
>            else
>                transaction = decode(line, Transaction + FN_OFF);
>        }
>
>        len = strlen(transaction);
>
>        if ((len > 1) && (line[2] != 'd'))
>            len--;      /* Non-directory;  don't compare last newline */
>
>        inserttransaction();
>
>        Transactions[iTransaction].percent = 0;
>        sTransaction = iTransaction;
>    }
>    else
>    if ((sTransaction < UNAVAILABLE) &&
>        (sscanf(line, " %26[0-9,.kKMGTPE]%hhd%%%lf%cB/s%*[ ]%zn%c",
>                Transactions[sTransaction].filesize,
>                &percent,
>                &Transactions[sTransaction].rate,
>                &Transactions[sTransaction].scale,
>                &len,
>                &Transactions[sTransaction].notime) == 5))
>    {
>        if (invalidprogress())
>        {
>            notices++;
>            printf("!line=%s", line);
>        }
>    }
>    else
>    {
>        flushtransactions();
>        fputs(line, stdout);
>    }
>
>done:
>    buffer[end] = savec;
>}
>
>int main(void)
>{
>    static size_t length;
>
>    /* Make `tail -f` sane */
>    (void) setlinebuf(stdout);
>
>#ifdef M_ARENA_MAX
>    /* Use a single arena */
>    (void) mallopt(M_ARENA_MAX, 1);
>#endif
>
>#ifdef M_MMAP_MAX
>    /* Disable use of mmap() to avoid wasteful pagesize alignments */
>    (void) mallopt(M_MMAP_MAX, 0);
>#endif
>
>#ifdef M_MXFAST
>    /* Don't use "fastbins" */
>    (void) mallopt(M_MXFAST, 0);
>#endif
>
>    while (true)
>    {
>        static size_t next, size;
>
>        if (next >= length)
>        {
>            static int eof;
>            size_t requested, completed;
>
>            if (eof)
>                break;
>
>            if (start > 0)
>            {
>                if ((next -= start) > 0)        /* Need to special-case */
>                    (void) memmove(buffer, buffer + start, next);
>
>                start = 0;
>            }
>            else
>            if (size == length)
>            {
>                static size_t sz;
>                char *savebuffer = buffer;
>
>                while ((buffer = realloc(buffer, sz += 8192)) == NULL)
>                {
>                    size_t need = sz;
>
>                    notices++;
>                    prefix = "!B";
>                    sz -= 8192;
>                    buffer = savebuffer;
>                    if (outofmemory(need))
>                    {
>                        if (sz == 0)
>                            die();
>                        print(length);
>                        start = next = 0;
>                        break;
>                    }
>                }
>
>                /* Allow for print()'s temporary NUL-termination */
>                size = sz - 1;
>            }
>
>            requested = size - next;
>            completed = fread(buffer + next, 1, requested, stdin);
>            length = next + completed;
>            if (completed < requested)
>            {
>                if (completed <= 0)
>                    break;
>
>                eof = feof(stdin) | ferror(stdin);
>            }
>        }
>
>        switch (buffer[next++])
>        {
>            case '\n':
>                print(next);
>                /* Fall through */
>
>            case '\r':
>                start = next;
>                /* Fall through */
>
>            default:
>                break;
>        }
>    }
>
>    if (start < length)
>        print(length);
>
>    flushtransactions();
>    free(Transactions);
>    free(buffer);
>
>    report();
>
>    return !!notices;
>}

Actions: View

Attachments on bug 6741: 18263 | 18425 | 18484