git.maemo.org Git - presencevnc/blob - libvnc/libvncclient/zrle.c

   1 /*
   2  *  Copyright (C) 2005 Johannes E. Schindelin.  All Rights Reserved.
   3  *
   4  *  This is free software; you can redistribute it and/or modify
   5  *  it under the terms of the GNU General Public License as published by
   6  *  the Free Software Foundation; either version 2 of the License, or
   7  *  (at your option) any later version.
   8  *
   9  *  This software is distributed in the hope that it will be useful,
  10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  *  GNU General Public License for more details.
  13  *
  14  *  You should have received a copy of the GNU General Public License
  15  *  along with this software; if not, write to the Free Software
  16  *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
  17  *  USA.
  18  */
  19
  20 #ifdef LIBVNCSERVER_HAVE_LIBZ
  21
  22 /*
  23  * zrle.c - handle zrle encoding.
  24  *
  25  * This file shouldn't be compiled directly.  It is included multiple times by
  26  * rfbproto.c, each time with a different definition of the macro BPP.  For
  27  * each value of BPP, this file defines a function which handles an zrle
  28  * encoded rectangle with BPP bits per pixel.
  29  */
  30
  31 #ifndef REALBPP
  32 #define REALBPP BPP
  33 #endif
  34
  35 #if !defined(UNCOMP) || UNCOMP==0
  36 #define HandleZRLE CONCAT2E(HandleZRLE,REALBPP)
  37 #define HandleZRLETile CONCAT2E(HandleZRLETile,REALBPP)
  38 #elif UNCOMP>0
  39 #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Down)
  40 #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Down)
  41 #else
  42 #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Up)
  43 #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Up)
  44 #endif
  45 #define CARDBPP CONCAT3E(uint,BPP,_t)
  46 #define CARDREALBPP CONCAT3E(uint,REALBPP,_t)
  47
  48 #define ENDIAN_LITTLE 0
  49 #define ENDIAN_BIG 1
  50 #define ENDIAN_NO 2
  51 #define ZYWRLE_ENDIAN ENDIAN_LITTLE
  52 #undef END_FIX
  53 #if ZYWRLE_ENDIAN == ENDIAN_LITTLE
  54 #  define END_FIX LE
  55 #elif ZYWRLE_ENDIAN == ENDIAN_BIG
  56 #  define END_FIX BE
  57 #else
  58 #  define END_FIX NE
  59 #endif
  60 #define __RFB_CONCAT3E(a,b,c) CONCAT3E(a,b,c)
  61 #define __RFB_CONCAT2E(a,b) CONCAT2E(a,b)
  62 #undef CPIXEL
  63 #if REALBPP != BPP
  64 #if UNCOMP == 0
  65 #define CPIXEL REALBPP
  66 #elif UNCOMP>0
  67 #define CPIXEL CONCAT2E(REALBPP,Down)
  68 #else
  69 #define CPIXEL CONCAT2E(REALBPP,Up)
  70 #endif
  71 #endif
  72 #define PIXEL_T __RFB_CONCAT3E(uint,BPP,_t)
  73 #if BPP!=8
  74 #define ZYWRLE_DECODE 1
  75 #include "../libvncserver/zywrletemplate.c"
  76 #endif
  77 #undef CPIXEL
  78
  79 static int HandleZRLETile(rfbClient* client,
  80         uint8_t* buffer,size_t buffer_length,
  81         int x,int y,int w,int h);
  82
  83 static rfbBool
  84 HandleZRLE (rfbClient* client, int rx, int ry, int rw, int rh)
  85 {
  86         rfbZRLEHeader header;
  87         int remaining;
  88         int inflateResult;
  89         int toRead;
  90         int min_buffer_size = rw * rh * (REALBPP / 8) * 2;
  91
  92         /* First make sure we have a large enough raw buffer to hold the
  93          * decompressed data.  In practice, with a fixed REALBPP, fixed frame
  94          * buffer size and the first update containing the entire frame
  95          * buffer, this buffer allocation should only happen once, on the
  96          * first update.
  97          */
  98         if ( client->raw_buffer_size < min_buffer_size) {
  99
 100                 if ( client->raw_buffer != NULL ) {
 101
 102                         free( client->raw_buffer );
 103
 104                 }
 105
 106                 client->raw_buffer_size = min_buffer_size;
 107                 client->raw_buffer = (char*) malloc( client->raw_buffer_size );
 108
 109         }
 110
 111         if (!ReadFromRFBServer(client, (char *)&header, sz_rfbZRLEHeader))
 112                 return FALSE;
 113
 114         remaining = rfbClientSwap32IfLE(header.length);
 115
 116         /* Need to initialize the decompressor state. */
 117         client->decompStream.next_in   = ( Bytef * )client->buffer;
 118         client->decompStream.avail_in  = 0;
 119         client->decompStream.next_out  = ( Bytef * )client->raw_buffer;
 120         client->decompStream.avail_out = client->raw_buffer_size;
 121         client->decompStream.data_type = Z_BINARY;
 122
 123         /* Initialize the decompression stream structures on the first invocation. */
 124         if ( client->decompStreamInited == FALSE ) {
 125
 126                 inflateResult = inflateInit( &client->decompStream );
 127
 128                 if ( inflateResult != Z_OK ) {
 129                         rfbClientLog(
 130                                         "inflateInit returned error: %d, msg: %s\n",
 131                                         inflateResult,
 132                                         client->decompStream.msg);
 133                         return FALSE;
 134                 }
 135
 136                 client->decompStreamInited = TRUE;
 137
 138         }
 139
 140         inflateResult = Z_OK;
 141
 142         /* Process buffer full of data until no more to process, or
 143          * some type of inflater error, or Z_STREAM_END.
 144          */
 145         while (( remaining > 0 ) &&
 146                         ( inflateResult == Z_OK )) {
 147
 148                 if ( remaining > RFB_BUFFER_SIZE ) {
 149                         toRead = RFB_BUFFER_SIZE;
 150                 }
 151                 else {
 152                         toRead = remaining;
 153                 }
 154
 155                 /* Fill the buffer, obtaining data from the server. */
 156                 if (!ReadFromRFBServer(client, client->buffer,toRead))
 157                         return FALSE;
 158
 159                 client->decompStream.next_in  = ( Bytef * )client->buffer;
 160                 client->decompStream.avail_in = toRead;
 161
 162                 /* Need to uncompress buffer full. */
 163                 inflateResult = inflate( &client->decompStream, Z_SYNC_FLUSH );
 164
 165                 /* We never supply a dictionary for compression. */
 166                 if ( inflateResult == Z_NEED_DICT ) {
 167                         rfbClientLog("zlib inflate needs a dictionary!\n");
 168                         return FALSE;
 169                 }
 170                 if ( inflateResult < 0 ) {
 171                         rfbClientLog(
 172                                         "zlib inflate returned error: %d, msg: %s\n",
 173                                         inflateResult,
 174                                         client->decompStream.msg);
 175                         return FALSE;
 176                 }
 177
 178                 /* Result buffer allocated to be at least large enough.  We should
 179                  * never run out of space!
 180                  */
 181                 if (( client->decompStream.avail_in > 0 ) &&
 182                                 ( client->decompStream.avail_out <= 0 )) {
 183                         rfbClientLog("zlib inflate ran out of space!\n");
 184                         return FALSE;
 185                 }
 186
 187                 remaining -= toRead;
 188
 189         } /* while ( remaining > 0 ) */
 190
 191         if ( inflateResult == Z_OK ) {
 192                 void* buf=client->raw_buffer;
 193                 int i,j;
 194
 195                 remaining = client->raw_buffer_size-client->decompStream.avail_out;
 196
 197                 for(j=0; j<rh; j+=rfbZRLETileHeight)
 198                         for(i=0; i<rw; i+=rfbZRLETileWidth) {
 199                                 int subWidth=(i+rfbZRLETileWidth>rw)?rw-i:rfbZRLETileWidth;
 200                                 int subHeight=(j+rfbZRLETileHeight>rh)?rh-j:rfbZRLETileHeight;
 201                                 int result=HandleZRLETile(client,buf,remaining,rx+i,ry+j,subWidth,subHeight);
 202
 203                                 if(result<0) {
 204                                         rfbClientLog("ZRLE decoding failed (%d)\n",result);
 205 return TRUE;
 206                                         return FALSE;
 207                                 }
 208
 209                                 buf+=result;
 210                                 remaining-=result;
 211                         }
 212         }
 213         else {
 214
 215                 rfbClientLog(
 216                                 "zlib inflate returned error: %d, msg: %s\n",
 217                                 inflateResult,
 218                                 client->decompStream.msg);
 219                 return FALSE;
 220
 221         }
 222
 223         return TRUE;
 224 }
 225
 226 #if REALBPP!=BPP && defined(UNCOMP) && UNCOMP!=0
 227 #if UNCOMP>0
 228 #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)>>UNCOMP)
 229 #else
 230 #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)<<(-(UNCOMP)))
 231 #endif
 232 #else
 233 #define UncompressCPixel(pointer) (*(CARDBPP*)pointer)
 234 #endif
 235
 236 static int HandleZRLETile(rfbClient* client,
 237                 uint8_t* buffer,size_t buffer_length,
 238                 int x,int y,int w,int h) {
 239         uint8_t* buffer_copy = buffer;
 240         uint8_t* buffer_end = buffer+buffer_length;
 241         uint8_t type;
 242         uint8_t zywrle_level = (client->appData.qualityLevel & 0x80) ?
 243                 0 : (3 - client->appData.qualityLevel / 3);
 244
 245         if(buffer_length<1)
 246                 return -2;
 247
 248         type = *buffer;
 249         buffer++;
 250         {
 251                 if( type == 0 ) /* raw */
 252 #if BPP!=8
 253           if( zywrle_level > 0 ){
 254                         CARDBPP* pFrame = (CARDBPP*)client->frameBuffer + y*client->width+x;
 255                         int ret;
 256                         client->appData.qualityLevel |= 0x80;
 257                         ret = HandleZRLETile(client, buffer, buffer_end-buffer, x, y, w, h);
 258                     client->appData.qualityLevel &= 0x7F;
 259                         if( ret < 0 ){
 260                                 return ret;
 261                         }
 262                         ZYWRLE_SYNTHESIZE( pFrame, pFrame, w, h, client->width, zywrle_level, (int*)client->zlib_buffer );
 263                         buffer += ret;
 264                   }else
 265 #endif
 266                 {
 267 #if REALBPP!=BPP
 268                         int i,j;
 269
 270                         if(1+w*h*REALBPP/8>buffer_length) {
 271                                 rfbClientLog("expected %d bytes, got only %d (%dx%d)\n",1+w*h*REALBPP/8,buffer_length,w,h);
 272                                 return -3;
 273                         }
 274
 275                         for(j=y*client->width; j<(y+h)*client->width; j+=client->width)
 276                                 for(i=x; i<x+w; i++,buffer+=REALBPP/8)
 277                                         ((CARDBPP*)client->frameBuffer)[j+i] = UncompressCPixel(buffer);
 278 #else
 279                         CopyRectangle(client, buffer, x, y, w, h);
 280                         buffer+=w*h*REALBPP/8;
 281 #endif
 282                 }
 283                 else if( type == 1 ) /* solid */
 284                 {
 285                         CARDBPP color = UncompressCPixel(buffer);
 286
 287                         if(1+REALBPP/8>buffer_length)
 288                                 return -4;
 289
 290                         FillRectangle(client, x, y, w, h, color);
 291
 292                         buffer+=REALBPP/8;
 293
 294                 }
 295                 else if( (type >= 2)&&(type <= 127) ) /* packed Palette */
 296                 {
 297                         CARDBPP palette[16];
 298                         int i,j,shift,
 299                                 bpp=(type>4?(type>16?8:4):(type>2?2:1)),
 300                                 mask=(1<<bpp)-1,
 301                                 divider=(8/bpp);
 302
 303                         if(1+type*REALBPP/8+((w+divider-1)/divider)*h>buffer_length)
 304                                 return -5;
 305
 306                         /* read palette */
 307                         for(i=0; i<type; i++,buffer+=REALBPP/8)
 308                                 palette[i] = UncompressCPixel(buffer);
 309
 310                         /* read palettized pixels */
 311                         for(j=y*client->width; j<(y+h)*client->width; j+=client->width) {
 312                                 for(i=x,shift=8-bpp; i<x+w; i++) {
 313                                         ((CARDBPP*)client->frameBuffer)[j+i] = palette[((*buffer)>>shift)&mask];
 314                                         shift-=bpp;
 315                                         if(shift<0) {
 316                                                 shift=8-bpp;
 317                                                 buffer++;
 318                                         }
 319                                 }
 320                                 if(shift<8-bpp)
 321                                         buffer++;
 322                         }
 323
 324                 }
 325                 /* case 17 ... 127: not used, but valid */
 326                 else if( type == 128 ) /* plain RLE */
 327                 {
 328                         int i=0,j=0;
 329                         while(j<h) {
 330                                 int color,length;
 331                                 /* read color */
 332                                 if(buffer+REALBPP/8+1>buffer_end)
 333                                         return -7;
 334                                 color = UncompressCPixel(buffer);
 335                                 buffer+=REALBPP/8;
 336                                 /* read run length */
 337                                 length=1;
 338                                 while(*buffer==0xff) {
 339                                         if(buffer+1>=buffer_end)
 340                                                 return -8;
 341                                         length+=*buffer;
 342                                         buffer++;
 343                                 }
 344                                 length+=*buffer;
 345                                 buffer++;
 346                                 while(j<h && length>0) {
 347                                         ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color;
 348                                         length--;
 349                                         i++;
 350                                         if(i>=w) {
 351                                                 i=0;
 352                                                 j++;
 353                                         }
 354                                 }
 355                                 if(length>0)
 356                                         rfbClientLog("Warning: possible ZRLE corruption\n");
 357                         }
 358
 359                 }
 360                 else if( type == 129 ) /* unused */
 361                 {
 362                         return -8;
 363                 }
 364                 else if( (type >= 130)&&(type <= 255) ) /* palette RLE */
 365                 {
 366                         CARDBPP palette[128];
 367                         int i,j;
 368
 369                         if(2+(type-128)*REALBPP/8>buffer_length)
 370                                 return -9;
 371
 372                         /* read palette */
 373                         for(i=0; i<type-128; i++,buffer+=REALBPP/8)
 374                                 palette[i] = UncompressCPixel(buffer);
 375                         /* read palettized pixels */
 376                         i=j=0;
 377                         while(j<h) {
 378                                 int color,length;
 379                                 /* read color */
 380                                 if(buffer>=buffer_end)
 381                                         return -10;
 382                                 color = palette[(*buffer)&0x7f];
 383                                 length=1;
 384                                 if(*buffer&0x80) {
 385                                         if(buffer+1>=buffer_end)
 386                                                 return -11;
 387                                         buffer++;
 388                                         /* read run length */
 389                                         while(*buffer==0xff) {
 390                                                 if(buffer+1>=buffer_end)
 391                                                         return -8;
 392                                                 length+=*buffer;
 393                                                 buffer++;
 394                                         }
 395                                         length+=*buffer;
 396                                 }
 397                                 buffer++;
 398                                 while(j<h && length>0) {
 399                                         ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color;
 400                                         length--;
 401                                         i++;
 402                                         if(i>=w) {
 403                                                 i=0;
 404                                                 j++;
 405                                         }
 406                                 }
 407                                 if(length>0)
 408                                         rfbClientLog("Warning: possible ZRLE corruption\n");
 409                         }
 410                 }
 411         }
 412
 413         return buffer-buffer_copy;
 414 }
 415
 416 #undef CARDBPP
 417 #undef CARDREALBPP
 418 #undef HandleZRLE
 419 #undef HandleZRLETile
 420 #undef UncompressCPixel
 421 #undef REALBPP
 422
 423 #endif
 424
 425 #undef UNCOMP