git.maemo.org Git - presencevnc/blob - libvnc/libvncserver/hextile.c

   1 /*
   2  * hextile.c
   3  *
   4  * Routines to implement Hextile Encoding
   5  */
   6
   7 /*
   8  *  OSXvnc Copyright (C) 2001 Dan McGuirk <mcguirk@incompleteness.net>.
   9  *  Original Xvnc code Copyright (C) 1999 AT&T Laboratories Cambridge.
  10  *  All Rights Reserved.
  11  *
  12  *  This is free software; you can redistribute it and/or modify
  13  *  it under the terms of the GNU General Public License as published by
  14  *  the Free Software Foundation; either version 2 of the License, or
  15  *  (at your option) any later version.
  16  *
  17  *  This software is distributed in the hope that it will be useful,
  18  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20  *  GNU General Public License for more details.
  21  *
  22  *  You should have received a copy of the GNU General Public License
  23  *  along with this software; if not, write to the Free Software
  24  *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
  25  *  USA.
  26  */
  27
  28 #include <rfb/rfb.h>
  29
  30 static rfbBool sendHextiles8(rfbClientPtr cl, int x, int y, int w, int h);
  31 static rfbBool sendHextiles16(rfbClientPtr cl, int x, int y, int w, int h);
  32 static rfbBool sendHextiles32(rfbClientPtr cl, int x, int y, int w, int h);
  33
  34
  35 /*
  36  * rfbSendRectEncodingHextile - send a rectangle using hextile encoding.
  37  */
  38
  39 rfbBool
  40 rfbSendRectEncodingHextile(rfbClientPtr cl,
  41                            int x,
  42                            int y,
  43                            int w,
  44                            int h)
  45 {
  46     rfbFramebufferUpdateRectHeader rect;
  47
  48     if (cl->ublen + sz_rfbFramebufferUpdateRectHeader > UPDATE_BUF_SIZE) {
  49         if (!rfbSendUpdateBuf(cl))
  50             return FALSE;
  51     }
  52
  53     rect.r.x = Swap16IfLE(x);
  54     rect.r.y = Swap16IfLE(y);
  55     rect.r.w = Swap16IfLE(w);
  56     rect.r.h = Swap16IfLE(h);
  57     rect.encoding = Swap32IfLE(rfbEncodingHextile);
  58
  59     memcpy(&cl->updateBuf[cl->ublen], (char *)&rect,
  60            sz_rfbFramebufferUpdateRectHeader);
  61     cl->ublen += sz_rfbFramebufferUpdateRectHeader;
  62
  63     rfbStatRecordEncodingSent(cl, rfbEncodingHextile,
  64           sz_rfbFramebufferUpdateRectHeader,
  65           sz_rfbFramebufferUpdateRectHeader + w * (cl->format.bitsPerPixel / 8) * h);
  66
  67     switch (cl->format.bitsPerPixel) {
  68     case 8:
  69         return sendHextiles8(cl, x, y, w, h);
  70     case 16:
  71         return sendHextiles16(cl, x, y, w, h);
  72     case 32:
  73         return sendHextiles32(cl, x, y, w, h);
  74     }
  75
  76     rfbLog("rfbSendRectEncodingHextile: bpp %d?\n", cl->format.bitsPerPixel);
  77     return FALSE;
  78 }
  79
  80
  81 #define PUT_PIXEL8(pix) (cl->updateBuf[cl->ublen++] = (pix))
  82
  83 #define PUT_PIXEL16(pix) (cl->updateBuf[cl->ublen++] = ((char*)&(pix))[0], \
  84                           cl->updateBuf[cl->ublen++] = ((char*)&(pix))[1])
  85
  86 #define PUT_PIXEL32(pix) (cl->updateBuf[cl->ublen++] = ((char*)&(pix))[0], \
  87                           cl->updateBuf[cl->ublen++] = ((char*)&(pix))[1], \
  88                           cl->updateBuf[cl->ublen++] = ((char*)&(pix))[2], \
  89                           cl->updateBuf[cl->ublen++] = ((char*)&(pix))[3])
  90
  91
  92 #define DEFINE_SEND_HEXTILES(bpp)                                               \
  93                                                                                 \
  94                                                                                 \
  95 static rfbBool subrectEncode##bpp(rfbClientPtr cli, uint##bpp##_t *data,        \
  96                 int w, int h, uint##bpp##_t bg, uint##bpp##_t fg, rfbBool mono);\
  97 static void testColours##bpp(uint##bpp##_t *data, int size, rfbBool *mono,      \
  98                   rfbBool *solid, uint##bpp##_t *bg, uint##bpp##_t *fg);        \
  99                                                                                 \
 100                                                                                 \
 101 /*                                                                              \
 102  * rfbSendHextiles                                                              \
 103  */                                                                             \
 104                                                                                 \
 105 static rfbBool                                                                  \
 106 sendHextiles##bpp(rfbClientPtr cl, int rx, int ry, int rw, int rh) {            \
 107     int x, y, w, h;                                                             \
 108     int startUblen;                                                             \
 109     char *fbptr;                                                                \
 110     uint##bpp##_t bg = 0, fg = 0, newBg, newFg;                                 \
 111     rfbBool mono, solid;                                                        \
 112     rfbBool validBg = FALSE;                                                    \
 113     rfbBool validFg = FALSE;                                                    \
 114     uint##bpp##_t clientPixelData[16*16*(bpp/8)];                               \
 115                                                                                 \
 116     for (y = ry; y < ry+rh; y += 16) {                                          \
 117         for (x = rx; x < rx+rw; x += 16) {                                      \
 118             w = h = 16;                                                         \
 119             if (rx+rw - x < 16)                                                 \
 120                 w = rx+rw - x;                                                  \
 121             if (ry+rh - y < 16)                                                 \
 122                 h = ry+rh - y;                                                  \
 123                                                                                 \
 124             if ((cl->ublen + 1 + (2 + 16 * 16) * (bpp/8)) >                     \
 125                 UPDATE_BUF_SIZE) {                                              \
 126                 if (!rfbSendUpdateBuf(cl))                                      \
 127                     return FALSE;                                               \
 128             }                                                                   \
 129                                                                                 \
 130             fbptr = (cl->scaledScreen->frameBuffer + (cl->scaledScreen->paddedWidthInBytes * y)   \
 131                      + (x * (cl->scaledScreen->bitsPerPixel / 8)));                   \
 132                                                                                 \
 133             (*cl->translateFn)(cl->translateLookupTable, &(cl->screen->serverFormat),      \
 134                                &cl->format, fbptr, (char *)clientPixelData,     \
 135                                cl->scaledScreen->paddedWidthInBytes, w, h);           \
 136                                                                                 \
 137             startUblen = cl->ublen;                                             \
 138             cl->updateBuf[startUblen] = 0;                                      \
 139             cl->ublen++;                                                        \
 140             rfbStatRecordEncodingSentAdd(cl, rfbEncodingHextile, 1);            \
 141                                                                                 \
 142             testColours##bpp(clientPixelData, w * h,                            \
 143                              &mono, &solid, &newBg, &newFg);                    \
 144                                                                                 \
 145             if (!validBg || (newBg != bg)) {                                    \
 146                 validBg = TRUE;                                                 \
 147                 bg = newBg;                                                     \
 148                 cl->updateBuf[startUblen] |= rfbHextileBackgroundSpecified;     \
 149                 PUT_PIXEL##bpp(bg);                                             \
 150             }                                                                   \
 151                                                                                 \
 152             if (solid) {                                                        \
 153                 continue;                                                       \
 154             }                                                                   \
 155                                                                                 \
 156             cl->updateBuf[startUblen] |= rfbHextileAnySubrects;                 \
 157                                                                                 \
 158             if (mono) {                                                         \
 159                 if (!validFg || (newFg != fg)) {                                \
 160                     validFg = TRUE;                                             \
 161                     fg = newFg;                                                 \
 162                     cl->updateBuf[startUblen] |= rfbHextileForegroundSpecified; \
 163                     PUT_PIXEL##bpp(fg);                                         \
 164                 }                                                               \
 165             } else {                                                            \
 166                 validFg = FALSE;                                                \
 167                 cl->updateBuf[startUblen] |= rfbHextileSubrectsColoured;        \
 168             }                                                                   \
 169                                                                                 \
 170             if (!subrectEncode##bpp(cl, clientPixelData, w, h, bg, fg, mono)) { \
 171                 /* encoding was too large, use raw */                           \
 172                 validBg = FALSE;                                                \
 173                 validFg = FALSE;                                                \
 174                 cl->ublen = startUblen;                                         \
 175                 cl->updateBuf[cl->ublen++] = rfbHextileRaw;                     \
 176                 (*cl->translateFn)(cl->translateLookupTable,                    \
 177                                    &(cl->screen->serverFormat), &cl->format, fbptr,        \
 178                                    (char *)clientPixelData,                     \
 179                                    cl->scaledScreen->paddedWidthInBytes, w, h); \
 180                                                                                 \
 181                 memcpy(&cl->updateBuf[cl->ublen], (char *)clientPixelData,      \
 182                        w * h * (bpp/8));                                        \
 183                                                                                 \
 184                 cl->ublen += w * h * (bpp/8);                                   \
 185                 rfbStatRecordEncodingSentAdd(cl, rfbEncodingHextile,            \
 186                              w * h * (bpp/8));                                  \
 187             }                                                                   \
 188         }                                                                       \
 189     }                                                                           \
 190                                                                                 \
 191     return TRUE;                                                                \
 192 }                                                                               \
 193                                                                                 \
 194                                                                                 \
 195 static rfbBool                                                                  \
 196 subrectEncode##bpp(rfbClientPtr cl, uint##bpp##_t *data, int w, int h,          \
 197                    uint##bpp##_t bg, uint##bpp##_t fg, rfbBool mono)            \
 198 {                                                                               \
 199     uint##bpp##_t cl2;                                                          \
 200     int x,y;                                                                    \
 201     int i,j;                                                                    \
 202     int hx=0,hy,vx=0,vy;                                                        \
 203     int hyflag;                                                                 \
 204     uint##bpp##_t *seg;                                                         \
 205     uint##bpp##_t *line;                                                        \
 206     int hw,hh,vw,vh;                                                            \
 207     int thex,they,thew,theh;                                                    \
 208     int numsubs = 0;                                                            \
 209     int newLen;                                                                 \
 210     int nSubrectsUblen;                                                         \
 211                                                                                 \
 212     nSubrectsUblen = cl->ublen;                                                 \
 213     cl->ublen++;                                                                \
 214     rfbStatRecordEncodingSentAdd(cl, rfbEncodingHextile, 1);                    \
 215                                                                                 \
 216     for (y=0; y<h; y++) {                                                       \
 217         line = data+(y*w);                                                      \
 218         for (x=0; x<w; x++) {                                                   \
 219             if (line[x] != bg) {                                                \
 220                 cl2 = line[x];                                                  \
 221                 hy = y-1;                                                       \
 222                 hyflag = 1;                                                     \
 223                 for (j=y; j<h; j++) {                                           \
 224                     seg = data+(j*w);                                           \
 225                     if (seg[x] != cl2) {break;}                                 \
 226                     i = x;                                                      \
 227                     while ((seg[i] == cl2) && (i < w)) i += 1;                  \
 228                     i -= 1;                                                     \
 229                     if (j == y) vx = hx = i;                                    \
 230                     if (i < vx) vx = i;                                         \
 231                     if ((hyflag > 0) && (i >= hx)) {                            \
 232                         hy += 1;                                                \
 233                     } else {                                                    \
 234                         hyflag = 0;                                             \
 235                     }                                                           \
 236                 }                                                               \
 237                 vy = j-1;                                                       \
 238                                                                                 \
 239                 /* We now have two possible subrects: (x,y,hx,hy) and           \
 240                  * (x,y,vx,vy).  We'll choose the bigger of the two.            \
 241                  */                                                             \
 242                 hw = hx-x+1;                                                    \
 243                 hh = hy-y+1;                                                    \
 244                 vw = vx-x+1;                                                    \
 245                 vh = vy-y+1;                                                    \
 246                                                                                 \
 247                 thex = x;                                                       \
 248                 they = y;                                                       \
 249                                                                                 \
 250                 if ((hw*hh) > (vw*vh)) {                                        \
 251                     thew = hw;                                                  \
 252                     theh = hh;                                                  \
 253                 } else {                                                        \
 254                     thew = vw;                                                  \
 255                     theh = vh;                                                  \
 256                 }                                                               \
 257                                                                                 \
 258                 if (mono) {                                                     \
 259                     newLen = cl->ublen - nSubrectsUblen + 2;                    \
 260                 } else {                                                        \
 261                     newLen = cl->ublen - nSubrectsUblen + bpp/8 + 2;            \
 262                 }                                                               \
 263                                                                                 \
 264                 if (newLen > (w * h * (bpp/8)))                                 \
 265                     return FALSE;                                               \
 266                                                                                 \
 267                 numsubs += 1;                                                   \
 268                                                                                 \
 269                 if (!mono) PUT_PIXEL##bpp(cl2);                                 \
 270                                                                                 \
 271                 cl->updateBuf[cl->ublen++] = rfbHextilePackXY(thex,they);       \
 272                 cl->updateBuf[cl->ublen++] = rfbHextilePackWH(thew,theh);       \
 273                 rfbStatRecordEncodingSentAdd(cl, rfbEncodingHextile, 1);        \
 274                                                                                 \
 275                 /*                                                              \
 276                  * Now mark the subrect as done.                                \
 277                  */                                                             \
 278                 for (j=they; j < (they+theh); j++) {                            \
 279                     for (i=thex; i < (thex+thew); i++) {                        \
 280                         data[j*w+i] = bg;                                       \
 281                     }                                                           \
 282                 }                                                               \
 283             }                                                                   \
 284         }                                                                       \
 285     }                                                                           \
 286                                                                                 \
 287     cl->updateBuf[nSubrectsUblen] = numsubs;                                    \
 288                                                                                 \
 289     return TRUE;                                                                \
 290 }                                                                               \
 291                                                                                 \
 292                                                                                 \
 293 /*                                                                              \
 294  * testColours() tests if there are one (solid), two (mono) or more             \
 295  * colours in a tile and gets a reasonable guess at the best background         \
 296  * pixel, and the foreground pixel for mono.                                    \
 297  */                                                                             \
 298                                                                                 \
 299 static void                                                                     \
 300 testColours##bpp(uint##bpp##_t *data, int size, rfbBool *mono, rfbBool *solid,  \
 301                  uint##bpp##_t *bg, uint##bpp##_t *fg) {                        \
 302     uint##bpp##_t colour1 = 0, colour2 = 0;                                     \
 303     int n1 = 0, n2 = 0;                                                         \
 304     *mono = TRUE;                                                               \
 305     *solid = TRUE;                                                              \
 306                                                                                 \
 307     for (; size > 0; size--, data++) {                                          \
 308                                                                                 \
 309         if (n1 == 0)                                                            \
 310             colour1 = *data;                                                    \
 311                                                                                 \
 312         if (*data == colour1) {                                                 \
 313             n1++;                                                               \
 314             continue;                                                           \
 315         }                                                                       \
 316                                                                                 \
 317         if (n2 == 0) {                                                          \
 318             *solid = FALSE;                                                     \
 319             colour2 = *data;                                                    \
 320         }                                                                       \
 321                                                                                 \
 322         if (*data == colour2) {                                                 \
 323             n2++;                                                               \
 324             continue;                                                           \
 325         }                                                                       \
 326                                                                                 \
 327         *mono = FALSE;                                                          \
 328         break;                                                                  \
 329     }                                                                           \
 330                                                                                 \
 331     if (n1 > n2) {                                                              \
 332         *bg = colour1;                                                          \
 333         *fg = colour2;                                                          \
 334     } else {                                                                    \
 335         *bg = colour2;                                                          \
 336         *fg = colour1;                                                          \
 337     }                                                                           \
 338 }
 339
 340 DEFINE_SEND_HEXTILES(8)
 341 DEFINE_SEND_HEXTILES(16)
 342 DEFINE_SEND_HEXTILES(32)