Attachment 18263 Details for Bug 6741 – rsync stdout filter

rsync stdout filter

postrsync.c (text/plain), 15.21 KB, created by Marc Aurèle La France on 2024-02-29 21:26:04 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Marc Aurèle La France

Created: 2024-02-29 21:26:04 UTC

Size: 15.21 KB

patch

obsolete

>/*
> * Copyright 2022-2024 by Marc AurÃ¨le La France, tsi@tuyoix.net
> *
> * Permission to use, copy, modify, distribute, and sell this software and its
> * documentation for any purpose is hereby granted without fee, provided that
> * the above copyright notice appear in all copies and that both that copyright
> * notice and this permission notice appear in supporting documentation, and
> * that the name of Marc AurÃ¨le La France not be used in advertising or
> * publicity pertaining to distribution of the software without specific,
> * written prior permission.  Marc AurÃ¨le La France makes no representations
> * about the suitability of this software for any purpose.  It is provided
> * "as-is" without express or implied warranty.
> *
> * MARC AURÃLE LA FRANCE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
> * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.  IN NO
> * EVENT SHALL MARC AURÃLE LA FRANCE BE LIABLE FOR ANY SPECIAL, INDIRECT OR
> * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
> * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
> * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
> * OF THIS SOFTWARE.
> */
>
>/* $TSI$ */
>
>/*
> * Post-process `rsync --out-format='-%i %n%L'` stdout to report file
> * transactions in ascending order (except for directory deletions which are
> * reported after the deletion of their contents).  This also strips out
> * interim progress lines given deletions may be reported between them.  The
> * file counts at the end of progress lines are ignored as they no longer make
> * sense after reordering transactions.
> *
> * This does not handle total transfer progress lines (i.e. --info=progress2,
> * specified or implied), nor progress lines generated by sending a signal to
> * rsync.  Probably other things too.
> *
> * The hyphen in front of the out-format makes locating %i output simpler and
> * more reliable.
> *
> * This should be statically linked to maximise heap space.  Out-of-memory
> * handling here may seem somewhat paranoid but is due to completeness in
> * dealing with allocation errors.  The latter almost always result in slightly
> * incorrect ordering instead of giving up.
> *
> * This is a reaction to https://bugzilla.samba.org/show_bug.cgi?id=6741 which
> * will never be addressed in rsync itself and probably shouldn't be.
> */
>
>/*
> * Revision history (from rsync's perspective):
> * - Clone stripcr, a filter that omits interim progress lines.
> * - Reorder deletions.
> * - Handle escaping of unusual filenames.
> * - Compensate for sort of directory contents that places non-directories
> *   ahead of subdirectories.  The resulting order is by pathname without
> *   regard for directory structure, meaning that "a/a" is reported after "a-"
> *   and before "a0".
> */
>
>#undef  _GNU_SOURCE
>#define _GNU_SOURCE 1   /* For reallocarray() */
>
>#include <limits.h>
>#include <signal.h>
>#include <stdbool.h>
>#include <stdio.h>
>#include <stdlib.h>
>#include <string.h>
>
>#undef  DELETING
>#define DELETING "-*deleting   "
>
>#undef  FN_OFF
>#define FN_OFF 13       /* Where %n starts in the out-format */
>
>#undef  MASK
>#define MASK ((sizeof(long double) << 1) - 1)   /* malloc() alignment */
>
>#undef  UNAVAILABLE
>#define UNAVAILABLE ((((size_t)(-1LL)) / sizeof(transaction_t)) + 1)
>
>/*
> * Until gcc quits whining about casts and assignments to void * pointers that
> * discard qualifiers.  If it ever does...  After all, void data cannot be
> * modified and are therefore implicitly const and no other qualifiers matter.
> */
>#ifndef voidconst
>#define voidconst /* const */
>#endif
>
>typedef struct
>{
>    voidconst char *encoded;
>    voidconst char *decoded;
>    double rate;
>    unsigned int hours;
>    unsigned char minutes;
>    unsigned char seconds;
>    char notime;
>    /* Enough for 64-bit values in decimal with commas and trailing NUL */
>    char filesize[27];
>    char percent;       /* Deletions are more than 100% */
>    char scale;
>} transaction_t;
>
>static transaction_t *Transactions;     /* Maintained in descending order */
>static size_t iTransaction;
>static size_t nTransaction, nTransactions;
>static size_t sTransaction = UNAVAILABLE;
>static voidconst char *Transaction, *transaction;
>
>static const char *line, *p, *prefix = "";
>
>static char *buffer;
>static size_t start, len;
>
>static char percent;
>static bool decoded;
>
>static size_t notices;
>
>static void report(void)
>{
>    printf("\n%zu notice%s issued!\n", notices, (notices == 1) ? "" : "s");
>}
>
>/* No sense in second-guessing malloc() internals.  So-o-o-o... */
>static void die(void)
>{
>    report();
>    puts("\n!!Premature termination.\n");
>    fflush(stdout);     /* Just in case */
>    /* Take down rsync as well */
>    kill(0, SIGTERM);   /* No return ... */
>    exit(SIGTERM);      /* ... supposedly ... */
>}
>
>static void printdeletion(const char *pathname)
>{
>    fputs(DELETING, stdout);
>    fputs(pathname, stdout);
>}
>
>static void printtransaction(void)
>{
>    voidconst char *Encoded = Transactions[nTransaction].encoded;
>
>    fputs(prefix, stdout);
>    prefix = "";
>
>    if (Transactions[nTransaction].percent > 100)
>    {
>        if (Encoded != Transactions[nTransaction].decoded)
>            free(Transactions[nTransaction].decoded);
>
>        printdeletion(Encoded);
>    }
>    else
>    {
>        if (sTransaction == nTransaction)
>            sTransaction = UNAVAILABLE;
>
>        if ((Encoded + FN_OFF) != Transactions[nTransaction].decoded)
>            free(Transactions[nTransaction].decoded);
>
>        fputs(Encoded, stdout);
>
>        if (Transactions[nTransaction].percent == 100)
>        {
>            printf("%15s 100%% %7.2f%cB/s ",
>                   Transactions[nTransaction].filesize,
>                   Transactions[nTransaction].rate,
>                   Transactions[nTransaction].scale);
>
>            if (Transactions[nTransaction].notime == '?')
>                puts("  ??:??:??");
>            else
>                printf("%4u:%02hhu:%02hhu\n",
>                       Transactions[nTransaction].hours,
>                       Transactions[nTransaction].minutes,
>                       Transactions[nTransaction].seconds);
>        }
>    }
>
>    free(Encoded);
>}
>
>static void flushtransactions(void)
>{
>    while (nTransaction-- > 0)
>        printtransaction();
>
>    nTransaction = 0;
>}
>
>static bool outofmemory(void)
>{
>    transaction_t *newTransactions;
>
>    if (nTransaction > 0)
>    {
>        nTransaction--;
>        printtransaction();
>        return false;
>    }
>
>    prefix = "";
>
>    if (nTransactions > 1)
>    {
>        Transactions = realloc(Transactions, sizeof(transaction_t));
>        nTransactions = 1;
>        return false;
>    }
>
>    if ((newTransactions = malloc(sizeof(transaction_t))) == NULL)
>        return true;
>
>    if ((nTransactions == 0) || (newTransactions < Transactions))
>    {
>        free(Transactions);
>        Transactions = newTransactions;
>        nTransactions = 1;
>        return false;
>    }
>
>    free(newTransactions);
>    return true;
>}
>
>static void inserttransaction(void)
>{
>    if (nTransaction >= nTransactions)
>    {
>        transaction_t *saveTransactions = Transactions;
>
>        if ((Transactions =
>                reallocarray(Transactions, sizeof(transaction_t),
>                             ++nTransactions)) == NULL)
>        {
>            if (--nTransactions == 0)
>                die();
>            notices++;
>            prefix = "!I";
>            Transactions = saveTransactions;
>            nTransaction--;
>            printtransaction();
>        }
>    }
>
>    for (iTransaction = nTransaction;  iTransaction-- > 0;  )
>    {
>        if (strncmp(transaction, Transactions[iTransaction].decoded, len) < 0)
>            break;
>
>        Transactions[iTransaction + 1] = Transactions[iTransaction];
>    }
>
>    iTransaction++;
>
>    Transactions[iTransaction].encoded = Transaction;
>    Transactions[iTransaction].decoded = transaction;
>    nTransaction++;
>}
>
>static voidconst char *decode(voidconst char *pathname)
>{
>    decoded = true;
>
>    for (p = pathname;  (p = strstr(p, "\\#")) != NULL;  p += 2)
>    {
>        char *filename, *dst;
>        const char *src;
>
>        if (((p[2] & 0xfc) != '0') ||
>            ((p[3] & 0xf8) != '0') ||
>            ((p[4] & 0xf8) != '0'))
>            continue;
>
>        len = strlen(pathname + ((5 - 1) - 1));
>        while ((filename = malloc(len)) == NULL)
>        {
>            notices++;
>            prefix = "!P";
>            if (outofmemory())
>            {
>                decoded = false;
>                return pathname;
>            }
>        }
>
>        src = pathname;
>        dst = filename;
>
>        while (src < p)
>            *dst++ = *src++;
>
>        while (true)
>        {
>            *dst++ = (src[2] << 6) | ((src[3] & 7) << 3) | (src[4] & 7);
>            src += 5;
>
>            while ((src[0] != '\\') ||
>                   (src[1] != '#') ||
>                   ((src[2] & 0xfc) != '0') ||
>                   ((src[3] & 0xf8) != '0') ||
>                   ((src[4] & 0xf8) != '0'))
>            {
>                if ((*dst++ = *src++) != '\0')
>                    continue;
>
>                len = (len & ~MASK) + !!(len & MASK);
>
>                if ((filename + len) <= dst)
>                    return filename;
>
>                return realloc(filename, dst - filename);
>            }
>        }
>    }
>
>    return pathname;
>}
>
>static bool printabletransaction(void)
>{
>    if (nTransaction == 0)
>        return false;
>
>    if (line[2] == 'd')
>        return true;    /* Directory */
>
>    if (sTransaction >= UNAVAILABLE)
>        return false;
>
>    if (strcmp(transaction, Transactions[sTransaction].decoded) <= 0)
>        return true;
>
>    if ((p = strrchr(Transactions[sTransaction].decoded, '/')) == NULL)
>        return false;
>
>    len = p + 1 - Transactions[sTransaction].decoded;
>    if (strncmp(transaction, Transactions[sTransaction].decoded, len) == 0)
>        return false;
>
>    return true;
>}
>
>static bool invalidprogress(void)
>{
>    if (Transactions[sTransaction].notime != '?')
>    {
>        char type;
>
>        if (sscanf(line + len,
>                   "%u:%2hhu:%2hhu%*[ ]%c",
>                   &Transactions[sTransaction].hours,
>                   &Transactions[sTransaction].minutes,
>                   &Transactions[sTransaction].seconds,
>                   &type) != 4)
>            return true;
>
>        if ((type != '\n') && (type != '('))
>            return true;
>
>        if ((Transactions[sTransaction].minutes >= 60) ||
>            (Transactions[sTransaction].seconds >= 60))
>            return true;
>    }
>
>    Transactions[sTransaction].percent = percent;
>
>    return false;
>}
>
>static void print(size_t end)
>{
>    char savec = buffer[end];
>
>    buffer[end] = '\0';
>    line = buffer + start;
>
>    /*
>     * Deletions may insert themselves ahead of other transfers or between
>     * iterations of progress lines.  Buffer them so they can be reinserted in
>     * the proper order, which is ascending by pathname except for directories
>     * which are to be reported after their contents have been reported.
>     */
>    if ((p = strstr(line, DELETING)) != NULL)
>    {
>        while ((Transaction = strdup(p + FN_OFF)) == NULL)
>        {
>            notices++;
>            prefix = "!D";
>            if (outofmemory())
>            {
>                fputs("!d", stdout);
>                printdeletion(p + FN_OFF);
>                goto done;
>            }
>        }
>
>        transaction = decode(Transaction);
>        len = strlen(transaction);
>
>        if ((len > 2) && (transaction[len - 2] == '/'))
>            len--;      /* Directory;  don't compare last newline */
>
>        inserttransaction();
>
>        Transactions[iTransaction].percent = SCHAR_MAX;
>
>        if (iTransaction <= sTransaction)
>            sTransaction++;
>    }
>    else
>    if (*line == '-')   /* -%i */
>    {
>        transaction = decode(line + FN_OFF);
>
>        if (printabletransaction())
>        {
>            while (nTransaction-- > 0)
>            {
>                if (strcmp(transaction, Transactions[nTransaction].decoded) < 0)
>                    break;
>
>                printtransaction();
>            }
>
>            nTransaction++;
>        }
>
>        while ((Transaction = strdup(line)) == NULL)
>        {
>            notices++;
>            prefix = "!N";
>            if (outofmemory())
>                die();
>        }
>
>        if (transaction == (line + FN_OFF))
>        {
>            if (decoded)
>                transaction = Transaction + FN_OFF;
>            else
>                transaction = decode(Transaction + FN_OFF);
>        }
>
>        len = strlen(transaction);
>
>        inserttransaction();
>
>        Transactions[iTransaction].percent = 0;
>        sTransaction = iTransaction;
>    }
>    else
>    if ((sTransaction < UNAVAILABLE) &&
>        (sscanf(line,
>                " %26[0-9,.kKMGTPE]%hhd%%%lf%cB/s%*[ ]%zn%c",
>                Transactions[sTransaction].filesize,
>                &percent,
>                &Transactions[sTransaction].rate,
>                &Transactions[sTransaction].scale,
>                &len,
>                &Transactions[sTransaction].notime) == 5))
>    {
>        if (invalidprogress())
>        {
>            notices++;
>            printf("!line=%s", line);
>        }
>    }
>    else
>    {
>        flushtransactions();
>        fputs(line, stdout);
>    }
>
>done:
>    buffer[end] = savec;
>}
>
>int main(void)
>{
>    static size_t length;
>
>    /* Make `tail -f` sane */
>    (void) setlinebuf(stdout);
>
>#ifdef M_MMAP_MAX
>    /* Disable use of mmap() to avoid wasteful pagesize alignments */
>    (void) mallopt(M_MMAP_MAX, 0);
>#endif
>
>    while (true)
>    {
>        static size_t next, size;
>
>        if (next >= length)
>        {
>            static int eof;
>            size_t requested, completed;
>
>            if (eof)
>                break;
>
>            if (start > 0)
>            {
>                if ((next -= start) > 0)        /* Need to special-case */
>                    (void) memmove(buffer, buffer + start, next);
>
>                start = 0;
>            }
>            else
>            if (size == length)
>            {
>                static size_t sz;
>                char *savebuffer = buffer;
>
>                while ((buffer = realloc(buffer, sz += 8192)) == NULL)
>                {
>                    notices++;
>                    prefix = "!B";
>                    sz -= 8192;
>                    buffer = savebuffer;
>                    if (outofmemory())
>                    {
>                        if (sz == 0)
>                            die();
>                        print(length);
>                        start = next = 0;
>                        break;
>                    }
>                }
>
>                /* Allow for print()'s temporary NUL-termination */
>                size = sz - 1;
>            }
>
>            requested = size - next;
>            completed = fread(buffer + next, 1, requested, stdin);
>            length = next + completed;
>            if (completed < requested)
>            {
>                if (completed <= 0)
>                    break;
>
>                eof = feof(stdin) | ferror(stdin);
>            }
>        }
>
>        switch (buffer[next++])
>        {
>            case '\n':
>                print(next);
>                /* Fall through */
>
>            case '\r':
>                start = next;
>                /* Fall through */
>
>            default:
>                break;
>        }
>    }
>
>    if (start < length)
>        print(length);
>
>    flushtransactions();
>    free(Transactions);
>    free(buffer);
>
>    report();
>
>    return !!notices;
>}

Actions: View

Attachments on bug 6741: 18263