textmode

usage:

sed -n '/pattern/='|yy092|sed -nf/dev/stdin file

       /*
         context split using sed 
       */
   
    int fileno (FILE *);
    int x,y,z;
   %option nounput noinput noyywrap 
   %%
   ^[0-9]+$ {
    if(x){fprintf(stdout,"%d,%dwxx%02d\n",z,atoi(yytext)-1,y);x=0;y++;}
    if(!x){z=atoi(yytext);x++;}
    }
   .|\n
   %%
    int main(){yylex();exit(0);}

      /* remove HTTP headers from multiple gzip or single zip from stdin */
    
     int fileno (FILE *);
     int setenv (const char *, const char *, int);
     #define jmp (yy_start) = 1 + 2 *
     int show_headers;
    %option nounput noinput noyywrap
    %%
    HTTP\/[01]\.[\15\12\40-\176]{0,1024}\r\n\r\n if(show_headers){fwrite(yytext,1,yyleng,yyout);putc(10,yyout);};
    .|\n if(!show_headers)fwrite(yytext,1,yyleng,yyout);
    %%
    int main(int argc,char *argv[])
    { 
    if(argc)if(argv[0])
    if(argv[1])show_headers++;
    yylex();
    exit(0);
    }

       /*
            experimental
            usage: [host=example.com] yy044 [ar]
       */
    int fileno (FILE *);
    int x,y,absolute,relative,omit,proto;
    int setenv (const char *, const char *, int);
    int fprintf(FILE *__restrict, const char *__restrict, ...);
    size_t strlen (const char *);
   http https://|http://
   js javascript:
   %option nounput noinput noyywrap 
   %%
   href=[^{http}{js}\"'][^\"'\40>]+ puts("yy044");
   
   href=[\"'][^{http}{js}][^\"'\40>]+ {
    omit=0;proto=0;
    if(yytext[6]==46)omit++;
    if(yytext[7]==47)proto++;
    if(!omit)if(relative)
    {
    y=0;for(x=6;x<yyleng;x++)
    {
    if(yytext[x]){yytext[y]=yytext[x];y++;};
    }
    if(proto)fwrite("https:",1,6,yyout);
    if(!proto)if(getenv("host")||getenv("Host"))fwrite("https://",1,8,yyout);
    if(!proto)if(getenv("host")){fwrite(getenv("host"),1,strlen(getenv("host")),yyout);goto x;};
    if(!proto)if(getenv("Host")){fwrite(getenv("Host"),1,strlen(getenv("Host")),yyout);goto x;};
    x:
    fwrite(yytext,1,yyleng-6,yyout);
    putc(10,stdout);
    }
    }
            /*
              javascript:void(0)
              #blah
            */
   "https://"[^\40<>\43\42\47|\r\n]+ {
    if(absolute)
    if(yyleng>8)
    {
    y=0;for(x=0;x<yyleng;x++){if(yytext[x])yytext[y]=yytext[x];y++;};
    fwrite(yytext,1,yyleng,yyout);
    putc(10,stdout);
    }
    }
   "http://"[^\40<>\43\42\47|\r\n]+ {
    if(absolute)
    if(yyleng>7)
    {
    y=0;for(x=0;x<yyleng;x++){if(yytext[x])yytext[y]=yytext[x];y++;};
    fwrite(yytext,1,yyleng,yyout);
    putc(10,stdout);
    }
    }
         /*
             href=https://x/http://
         */
   .|\n
   %%
          int main(int argc,char* argv[])
          { 
          absolute++;relative++;
          if(argc)if(argv[1])
          {
          if(argv[1][0]==97)relative--;
          if(argv[1][0]==114)absolute--;
          }
          yylex();
          exit(0);
          }

   /* remove HTTP headers from multiple gzip or single zip from stdin */

    int fileno(FILE *); 
    #define jmp (yy_start) = 1 + 2 *
    #define echo do{if(fwrite(yytext,(size_t)yyleng,1,yyout)){}}while(0)

   xa "\x1f\x8b"    
   /* GZIP magic number */
   xb "\x50\x4B\x03\x04"
   /* ZIP magic number */

   %s xa
   %option noyywrap noinput nounput
   %%
   {xa}|{xb} echo;jmp xa;
   <xa>"HTTP" jmp 0;
   <xa>.|\n echo;
   \n|.
   %%
   int main(){ yylex();exit(0);}

Solution:

       #!/bin/sh
       # usage: $0 pattern < file
       x=$(echo x|tr x '\34');
       tr -d '\12'|sed "s/$x//g;s/$1/$x&$x/g"|tr $x '\12'|sed -n "/^$1/p"

Retrieving 100s of small JPEG files with a single TCP connection. Here, 2 TCP connections for 141 files.

Splitting the output into image files and making a PDF from the images.

yy025 can be found here:

https://news.ycombinator.com/item?id=27178484

https://news.ycombinator.com/item?id=27180125

yy056 can be found here:

https://news.ycombinator.com/item?id=27769305

    # std utils: split, seq, od, tr, sed, flex
    # common utils: nc, xxd, imagemagick
    # simple utils made with flex: yy025, yy056

    (export Connection=keep-alive
    x1=http://www.minimizedistraction.com/img/vrg_google_doc_final_vrs03-
    x2(){ seq -f "$x1%g.jpg" $1 $2;};
    x3(){ yy025|nc -vvn 173.236.175.199 80;};
    x2   1 100|x3;
    x2 101 200|x3;
    )|exec yy056|exec od -An -tx1 -vw99999|exec tr -d '\40'|exec sed 's/ffd9ffd8/ffd9\
    ffd8/g'|exec sed -n /ffd8/p|exec split -l1;
    for x in x??;do xxd -p -r < $x > $x.jpg;rm $x;done;
    convert x??.jpg 1.pdf 2>/dev/null;rm x??.jpg

    firefox ./1.pdf

      cat > 056.l
 
        /* JPEG file carver */
    
     #define echo ECHO
     #define jmp BEGIN
     int fileno(FILE *);
    
    xa "\xFF\xD8"
    xb "\xFF\xD9" 
    
    %s xa 
    %option noyywrap nounput noinput
    %%
    
    {xa} echo;jmp xa;
    <xa>{xb} echo;jmp 0;
    <xa>.|\n|\r echo;
    .|\n
    
    %%
    int main(){ yylex();exit(0) ;}
    
    ^D

    flex 056.l
    cc -static -o yy056 lex.yy.c

    cat > 054.l;    
     int fileno(FILE *);
     #define jmp BEGIN
     #define echo ECHO
    
    xa "\x1f\x8b"    
     /* GZIP magic number */
    
    %s xa
    %option noyywrap noinput nounput
    %%
    {xa} echo;jmp xa;
    <xa>.|\n echo;
    \n|.
    %%
    int main(){ yylex();exit(0);}
    ^D

    flex -8Crf 054.l
    cc -pipe -static lex.yy.c -o yy054

    example usage:
    printf 'GET /file.gz\r\nHost: example.com\r\nConnection: close\r\n\r\n" |nc -vv example.com 80|yy054|zcat

    static int hexdigit(char x)
    {
      if (x >= '0' && x <= '9') return x - '0';
      if (x >= 'a' && x <= 'f') return 10 + (x - 'a');
      if (x >= 'A' && x <= 'F') return 10 + (x - 'A');
      return -1;
    }
    int hexparse(unsigned char *y,long long len,const char *x)
    {
      if (!x) return 0;
      while (len > 0) {
        int digit0;
        int digit1;
        digit0 = hexdigit(x[0]); if (digit0 == -1) return 0;
        digit1 = hexdigit(x[1]); if (digit1 == -1) return 0;
        *y++ = digit1 + 16 * digit0;
        --len;
        x += 2;
      }
      if (x[0]) return 0;
      return 1;
    }
    char buf[256];
    long long buflen = 0;
    static void flush(void) {
        if (writeall(1, buf, buflen) == -1) _exit(errno);
        buflen = 0;
    }
    static void xeet(void) {
        flush();
        if (fsyncfd(1) == -1) _exit(errno);
        _exit(0);
    }
    static void wrch(const char ch) {
        if (buflen >= sizeof buf) flush();
        buf[buflen++] = ch;
        return;
    }
    char inbuf[128];
    int main(int argc, char **argv) {
        long long r, i;
        for (;;) {
            r = read(0, inbuf, sizeof inbuf);
            if (r == -1) _exit(errno);
            if (r == 0) break;
    
            for (i = 0; i < r; ++i) {
                wrch("0123456789abcdef"[15 & (inbuf[i] >> 4)]);
                wrch("0123456789abcdef"[15 & inbuf[i]]);
            }
        }
        wrch('\n');
        xeet();
        return 0;
    }

    #include <unistd.h>
    #include <poll.h>
    #include <errno.h>
    #include <sys/types.h>
    #include <sys/stat.h>
    #ifndef EINTR
    #define EINTR (-5004)
    #endif
    #ifndef EAGAIN
    #define EAGAIN (-5011)
    #endif
    #ifndef EWOULDBLOCK
    #define EWOULDBLOCK (-7011)
    #endif
    int writeall(int fd,const void *xv,long long xlen)
    {
      const unsigned char *x = xv;
      long long w;
      while (xlen > 0) {
        w = xlen;
        if (w > 1048576) w = 1048576;
        w = write(fd,x,w);
        if (w < 0) {
          if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
            struct pollfd p;
            p.fd = fd;
            p.events = POLLOUT | POLLERR;
            poll(&p,1,-1);
            continue;
          }
          return -1;
        }
        x += w;
        xlen -= w;
      }
      return 0;
    }
    long long readblock(int fd, void *x, long long xlen) {
        long long r;
        char *buf = x;
        while (xlen > 0) {
            r = xlen;
            if (r > 1048576) r = 1048576;
            r = read(fd, buf, r);
            if (r == 0) break;
            if (r == -1) {
                if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
                    struct pollfd p;
                    p.fd = fd;
                    p.events = POLLIN;
                    poll(&p, 1, -1);
                    continue;
                }
                return -1;
            }
            buf += r; xlen -= r;
        }
        return (buf - (char *)x);
    }
    int fsyncfd(int fd) {
        struct stat st;
        if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
            if (fsync(fd) == -1) return -1;
        }
        return 0;
    }

      # make new http client (yy025)
      # hh2 is an #include in 025.l below
      yy018 < ua.txt > hh2;
      flex -8iCrfa 025.l;
      # cpp -P 025.l|less;
      cc -Wall -std=c89 -static -o yy025 lex.yy.c;

   save as "025.l":

        int yy_get_next_buffer();
        int fileno(FILE *);
        int setenv (const char *, const char *, int);
        int dprintf(int, const char *__restrict, ...);
        #define httpMethod "GET"
        #define httpVersion "1.1"
        #define Host ""
        #define jmp BEGIN
        #define Y(x,y) fprintf(stdout,x,y)
        int count,path,ka;
        int httpheaders(){
          setenv("httpMethod",httpMethod,0);Y("%s ",getenv("httpMethod"));
          Y("%s HTTP/",getenv("Path"));
          setenv("httpVersion",httpVersion,0);Y("%s\r\n",getenv("httpVersion"));
          #include "hh2"                                                                                                                         
          fputs("\r\n",stdout);
          return 0;}
        int keepaliveclose(){setenv("Connection","close",1);return 0;}
     %option nounput 
     %s xa xb xc
     xa "http://"|"https://"
     xb [-A-Za-z0-9.:]*
     xc [^#'|<> \r\n]*
     %%
     ^{xa} count++;setenv("Scheme",yytext,0);jmp xa;
     <xa>{xb} setenv("Host",yytext,1);if(!getenv("Host"))setenv("Host",Host,0);jmp xb;
     <xb>\n path=0;setenv("Path","/",0);httpheaders();jmp 0;
     <xb>{xc} path=1;setenv("Path",yytext,1);httpheaders();jmp 0;
     .|\n
     %%
            int main(){input();yylex();if(!getenv("nokac")){if(count>1){keepaliveclose();httpheaders();};}return 0;}
            int yywrap(){dprintf(6,"%s",getenv("Host"));return 0;}