2 * Copyright (C) 2000, 2001 Const Kaplinsky. All Rights Reserved.
4 * This is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This software is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this software; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
20 #ifdef LIBVNCSERVER_HAVE_LIBZ
21 #ifdef LIBVNCSERVER_HAVE_LIBJPEG
24 * tight.c - handle ``tight'' encoding.
26 * This file shouldn't be compiled directly. It is included multiple
27 * times by rfbproto.c, each time with a different definition of the
28 * macro BPP. For each value of BPP, this file defines a function
29 * which handles a tight-encoded rectangle with BPP bits per pixel.
33 #define TIGHT_MIN_TO_COMPRESS 12
35 #define CARDBPP CONCAT3E(uint,BPP,_t)
36 #define filterPtrBPP CONCAT2E(filterPtr,BPP)
38 #define HandleTightBPP CONCAT2E(HandleTight,BPP)
39 #define InitFilterCopyBPP CONCAT2E(InitFilterCopy,BPP)
40 #define InitFilterPaletteBPP CONCAT2E(InitFilterPalette,BPP)
41 #define InitFilterGradientBPP CONCAT2E(InitFilterGradient,BPP)
42 #define FilterCopyBPP CONCAT2E(FilterCopy,BPP)
43 #define FilterPaletteBPP CONCAT2E(FilterPalette,BPP)
44 #define FilterGradientBPP CONCAT2E(FilterGradient,BPP)
47 #define DecompressJpegRectBPP CONCAT2E(DecompressJpegRect,BPP)
52 #define RGB_TO_PIXEL(bpp,r,g,b) \
53 (((CARD##bpp)(r) & client->format.redMax) << client->format.redShift | \
54 ((CARD##bpp)(g) & client->format.greenMax) << client->format.greenShift | \
55 ((CARD##bpp)(b) & client->format.blueMax) << client->format.blueShift)
57 #define RGB24_TO_PIXEL(bpp,r,g,b) \
58 ((((CARD##bpp)(r) & 0xFF) * client->format.redMax + 127) / 255 \
59 << client->format.redShift | \
60 (((CARD##bpp)(g) & 0xFF) * client->format.greenMax + 127) / 255 \
61 << client->format.greenShift | \
62 (((CARD##bpp)(b) & 0xFF) * client->format.blueMax + 127) / 255 \
63 << client->format.blueShift)
65 #define RGB24_TO_PIXEL32(r,g,b) \
66 (((uint32_t)(r) & 0xFF) << client->format.redShift | \
67 ((uint32_t)(g) & 0xFF) << client->format.greenShift | \
68 ((uint32_t)(b) & 0xFF) << client->format.blueShift)
72 /* Type declarations */
74 typedef void (*filterPtrBPP)(rfbClient* client, int, CARDBPP *);
78 static int InitFilterCopyBPP (rfbClient* client, int rw, int rh);
79 static int InitFilterPaletteBPP (rfbClient* client, int rw, int rh);
80 static int InitFilterGradientBPP (rfbClient* client, int rw, int rh);
81 static void FilterCopyBPP (rfbClient* client, int numRows, CARDBPP *destBuffer);
82 static void FilterPaletteBPP (rfbClient* client, int numRows, CARDBPP *destBuffer);
83 static void FilterGradientBPP (rfbClient* client, int numRows, CARDBPP *destBuffer);
86 static rfbBool DecompressJpegRectBPP(rfbClient* client, int x, int y, int w, int h);
92 HandleTightBPP (rfbClient* client, int rx, int ry, int rw, int rh)
97 filterPtrBPP filterFn;
100 int err, stream_id, compressedLen, bitsPixel;
101 int bufferSize, rowSize, numRows, portionLen, rowsProcessed, extraBytes;
103 if (!ReadFromRFBServer(client, (char *)&comp_ctl, 1))
106 /* Flush zlib streams if we are told by the server to do so. */
107 for (stream_id = 0; stream_id < 4; stream_id++) {
108 if ((comp_ctl & 1) && client->zlibStreamActive[stream_id]) {
109 if (inflateEnd (&client->zlibStream[stream_id]) != Z_OK &&
110 client->zlibStream[stream_id].msg != NULL)
111 rfbClientLog("inflateEnd: %s\n", client->zlibStream[stream_id].msg);
112 client->zlibStreamActive[stream_id] = FALSE;
117 /* Handle solid rectangles. */
118 if (comp_ctl == rfbTightFill) {
120 if (client->format.depth == 24 && client->format.redMax == 0xFF &&
121 client->format.greenMax == 0xFF && client->format.blueMax == 0xFF) {
122 if (!ReadFromRFBServer(client, client->buffer, 3))
124 fill_colour = RGB24_TO_PIXEL32(client->buffer[0], client->buffer[1], client->buffer[2]);
126 if (!ReadFromRFBServer(client, (char*)&fill_colour, sizeof(fill_colour)))
130 if (!ReadFromRFBServer(client, (char*)&fill_colour, sizeof(fill_colour)))
134 FillRectangle(client, rx, ry, rw, rh, fill_colour);
140 if (comp_ctl == rfbTightJpeg) {
141 rfbClientLog("Tight encoding: JPEG is not supported in 8 bpp mode.\n");
145 if (comp_ctl == rfbTightJpeg) {
146 return DecompressJpegRectBPP(client, rx, ry, rw, rh);
150 /* Quit on unsupported subencoding value. */
151 if (comp_ctl > rfbTightMaxSubencoding) {
152 rfbClientLog("Tight encoding: bad subencoding value received.\n");
157 * Here primary compression mode handling begins.
158 * Data was processed with optional filter + zlib compression.
161 /* First, we should identify a filter to use. */
162 if ((comp_ctl & rfbTightExplicitFilter) != 0) {
163 if (!ReadFromRFBServer(client, (char*)&filter_id, 1))
167 case rfbTightFilterCopy:
168 filterFn = FilterCopyBPP;
169 bitsPixel = InitFilterCopyBPP(client, rw, rh);
171 case rfbTightFilterPalette:
172 filterFn = FilterPaletteBPP;
173 bitsPixel = InitFilterPaletteBPP(client, rw, rh);
175 case rfbTightFilterGradient:
176 filterFn = FilterGradientBPP;
177 bitsPixel = InitFilterGradientBPP(client, rw, rh);
180 rfbClientLog("Tight encoding: unknown filter code received.\n");
184 filterFn = FilterCopyBPP;
185 bitsPixel = InitFilterCopyBPP(client, rw, rh);
187 if (bitsPixel == 0) {
188 rfbClientLog("Tight encoding: error receiving palette.\n");
192 /* Determine if the data should be decompressed or just copied. */
193 rowSize = (rw * bitsPixel + 7) / 8;
194 if (rh * rowSize < TIGHT_MIN_TO_COMPRESS) {
195 if (!ReadFromRFBServer(client, (char*)client->buffer, rh * rowSize))
198 buffer2 = &client->buffer[TIGHT_MIN_TO_COMPRESS * 4];
199 filterFn(client, rh, (CARDBPP *)buffer2);
201 CopyRectangle(client, (uint8_t *)buffer2, rx, ry, rw, rh);
206 /* Read the length (1..3 bytes) of compressed data following. */
207 compressedLen = (int)ReadCompactLen(client);
208 if (compressedLen <= 0) {
209 rfbClientLog("Incorrect data received from the server.\n");
213 /* Now let's initialize compression stream if needed. */
214 stream_id = comp_ctl & 0x03;
215 zs = &client->zlibStream[stream_id];
216 if (!client->zlibStreamActive[stream_id]) {
220 err = inflateInit(zs);
223 rfbClientLog("InflateInit error: %s.\n", zs->msg);
226 client->zlibStreamActive[stream_id] = TRUE;
229 /* Read, decode and draw actual pixel data in a loop. */
231 bufferSize = RFB_BUFFER_SIZE * bitsPixel / (bitsPixel + BPP) & 0xFFFFFFFC;
232 buffer2 = &client->buffer[bufferSize];
233 if (rowSize > bufferSize) {
234 /* Should be impossible when RFB_BUFFER_SIZE >= 16384 */
235 rfbClientLog("Internal error: incorrect buffer size.\n");
242 while (compressedLen > 0) {
243 if (compressedLen > ZLIB_BUFFER_SIZE)
244 portionLen = ZLIB_BUFFER_SIZE;
246 portionLen = compressedLen;
248 if (!ReadFromRFBServer(client, (char*)client->zlib_buffer, portionLen))
251 compressedLen -= portionLen;
253 zs->next_in = (Bytef *)client->zlib_buffer;
254 zs->avail_in = portionLen;
257 zs->next_out = (Bytef *)&client->buffer[extraBytes];
258 zs->avail_out = bufferSize - extraBytes;
260 err = inflate(zs, Z_SYNC_FLUSH);
261 if (err == Z_BUF_ERROR) /* Input exhausted -- no problem. */
263 if (err != Z_OK && err != Z_STREAM_END) {
264 if (zs->msg != NULL) {
265 rfbClientLog("Inflate error: %s.\n", zs->msg);
267 rfbClientLog("Inflate error: %d.\n", err);
272 numRows = (bufferSize - zs->avail_out) / rowSize;
274 filterFn(client, numRows, (CARDBPP *)buffer2);
276 extraBytes = bufferSize - zs->avail_out - numRows * rowSize;
278 memcpy(client->buffer, &client->buffer[numRows * rowSize], extraBytes);
280 CopyRectangle(client, (uint8_t *)buffer2, rx, ry+rowsProcessed, rw, numRows);
282 rowsProcessed += numRows;
284 while (zs->avail_out == 0);
287 if (rowsProcessed != rh) {
288 rfbClientLog("Incorrect number of scan lines after decompression.\n");
295 /*----------------------------------------------------------------------------
302 InitFilterCopyBPP (rfbClient* client, int rw, int rh)
304 client->rectWidth = rw;
307 if (client->format.depth == 24 && client->format.redMax == 0xFF &&
308 client->format.greenMax == 0xFF && client->format.blueMax == 0xFF) {
309 client->cutZeros = TRUE;
312 client->cutZeros = FALSE;
320 FilterCopyBPP (rfbClient* client, int numRows, CARDBPP *dst)
326 if (client->cutZeros) {
327 for (y = 0; y < numRows; y++) {
328 for (x = 0; x < client->rectWidth; x++) {
329 dst[y*client->rectWidth+x] =
330 RGB24_TO_PIXEL32(client->buffer[(y*client->rectWidth+x)*3],
331 client->buffer[(y*client->rectWidth+x)*3+1],
332 client->buffer[(y*client->rectWidth+x)*3+2]);
339 memcpy (dst, client->buffer, numRows * client->rectWidth * (BPP / 8));
343 InitFilterGradientBPP (rfbClient* client, int rw, int rh)
347 bits = InitFilterCopyBPP(client, rw, rh);
348 if (client->cutZeros)
349 memset(client->tightPrevRow, 0, rw * 3);
351 memset(client->tightPrevRow, 0, rw * 3 * sizeof(uint16_t));
359 FilterGradient24 (rfbClient* client, int numRows, uint32_t *dst)
362 uint8_t thisRow[2048*3];
366 for (y = 0; y < numRows; y++) {
368 /* First pixel in a row */
369 for (c = 0; c < 3; c++) {
370 pix[c] = client->tightPrevRow[c] + client->buffer[y*client->rectWidth*3+c];
373 dst[y*client->rectWidth] = RGB24_TO_PIXEL32(pix[0], pix[1], pix[2]);
375 /* Remaining pixels of a row */
376 for (x = 1; x < client->rectWidth; x++) {
377 for (c = 0; c < 3; c++) {
378 est[c] = (int)client->tightPrevRow[x*3+c] + (int)pix[c] -
379 (int)client->tightPrevRow[(x-1)*3+c];
382 } else if (est[c] < 0x00) {
385 pix[c] = (uint8_t)est[c] + client->buffer[(y*client->rectWidth+x)*3+c];
386 thisRow[x*3+c] = pix[c];
388 dst[y*client->rectWidth+x] = RGB24_TO_PIXEL32(pix[0], pix[1], pix[2]);
391 memcpy(client->tightPrevRow, thisRow, client->rectWidth * 3);
398 FilterGradientBPP (rfbClient* client, int numRows, CARDBPP *dst)
401 CARDBPP *src = (CARDBPP *)client->buffer;
402 uint16_t *thatRow = (uint16_t *)client->tightPrevRow;
403 uint16_t thisRow[2048*3];
410 if (client->cutZeros) {
411 FilterGradient24(client, numRows, dst);
416 max[0] = client->format.redMax;
417 max[1] = client->format.greenMax;
418 max[2] = client->format.blueMax;
420 shift[0] = client->format.redShift;
421 shift[1] = client->format.greenShift;
422 shift[2] = client->format.blueShift;
424 for (y = 0; y < numRows; y++) {
426 /* First pixel in a row */
427 for (c = 0; c < 3; c++) {
428 pix[c] = (uint16_t)(((src[y*client->rectWidth] >> shift[c]) + thatRow[c]) & max[c]);
431 dst[y*client->rectWidth] = RGB_TO_PIXEL(BPP, pix[0], pix[1], pix[2]);
433 /* Remaining pixels of a row */
434 for (x = 1; x < client->rectWidth; x++) {
435 for (c = 0; c < 3; c++) {
436 est[c] = (int)thatRow[x*3+c] + (int)pix[c] - (int)thatRow[(x-1)*3+c];
437 if (est[c] > (int)max[c]) {
438 est[c] = (int)max[c];
439 } else if (est[c] < 0) {
442 pix[c] = (uint16_t)(((src[y*client->rectWidth+x] >> shift[c]) + est[c]) & max[c]);
443 thisRow[x*3+c] = pix[c];
445 dst[y*client->rectWidth+x] = RGB_TO_PIXEL(BPP, pix[0], pix[1], pix[2]);
447 memcpy(thatRow, thisRow, client->rectWidth * 3 * sizeof(uint16_t));
452 InitFilterPaletteBPP (rfbClient* client, int rw, int rh)
457 CARDBPP *palette = (CARDBPP *)client->tightPalette;
460 client->rectWidth = rw;
462 if (!ReadFromRFBServer(client, (char*)&numColors, 1))
465 client->rectColors = (int)numColors;
466 if (++client->rectColors < 2)
470 if (client->format.depth == 24 && client->format.redMax == 0xFF &&
471 client->format.greenMax == 0xFF && client->format.blueMax == 0xFF) {
472 if (!ReadFromRFBServer(client, (char*)&client->tightPalette, client->rectColors * 3))
474 for (i = client->rectColors - 1; i >= 0; i--) {
475 palette[i] = RGB24_TO_PIXEL32(client->tightPalette[i*3],
476 client->tightPalette[i*3+1],
477 client->tightPalette[i*3+2]);
479 return (client->rectColors == 2) ? 1 : 8;
483 if (!ReadFromRFBServer(client, (char*)&client->tightPalette, client->rectColors * (BPP / 8)))
486 return (client->rectColors == 2) ? 1 : 8;
490 FilterPaletteBPP (rfbClient* client, int numRows, CARDBPP *dst)
493 uint8_t *src = (uint8_t *)client->buffer;
494 CARDBPP *palette = (CARDBPP *)client->tightPalette;
496 if (client->rectColors == 2) {
497 w = (client->rectWidth + 7) / 8;
498 for (y = 0; y < numRows; y++) {
499 for (x = 0; x < client->rectWidth / 8; x++) {
500 for (b = 7; b >= 0; b--)
501 dst[y*client->rectWidth+x*8+7-b] = palette[src[y*w+x] >> b & 1];
503 for (b = 7; b >= 8 - client->rectWidth % 8; b--) {
504 dst[y*client->rectWidth+x*8+7-b] = palette[src[y*w+x] >> b & 1];
508 for (y = 0; y < numRows; y++)
509 for (x = 0; x < client->rectWidth; x++)
510 dst[y*client->rectWidth+x] = palette[(int)src[y*client->rectWidth+x]];
516 /*----------------------------------------------------------------------------
518 * JPEG decompression.
523 DecompressJpegRectBPP(rfbClient* client, int x, int y, int w, int h)
525 struct jpeg_decompress_struct cinfo;
526 struct jpeg_error_mgr jerr;
528 uint8_t *compressedData;
530 JSAMPROW rowPointer[1];
533 compressedLen = (int)ReadCompactLen(client);
534 if (compressedLen <= 0) {
535 rfbClientLog("Incorrect data received from the server.\n");
539 compressedData = malloc(compressedLen);
540 if (compressedData == NULL) {
541 rfbClientLog("Memory allocation error.\n");
545 if (!ReadFromRFBServer(client, (char*)compressedData, compressedLen)) {
546 free(compressedData);
550 cinfo.err = jpeg_std_error(&jerr);
551 cinfo.client_data = client;
552 jpeg_create_decompress(&cinfo);
554 JpegSetSrcManager(&cinfo, compressedData, compressedLen);
556 jpeg_read_header(&cinfo, TRUE);
557 cinfo.out_color_space = JCS_RGB;
559 jpeg_start_decompress(&cinfo);
560 if (cinfo.output_width != w || cinfo.output_height != h ||
561 cinfo.output_components != 3) {
562 rfbClientLog("Tight Encoding: Wrong JPEG data received.\n");
563 jpeg_destroy_decompress(&cinfo);
564 free(compressedData);
568 rowPointer[0] = (JSAMPROW)client->buffer;
570 while (cinfo.output_scanline < cinfo.output_height) {
571 jpeg_read_scanlines(&cinfo, rowPointer, 1);
572 if (client->jpegError) {
575 pixelPtr = (CARDBPP *)&client->buffer[RFB_BUFFER_SIZE / 2];
576 for (dx = 0; dx < w; dx++) {
578 RGB24_TO_PIXEL(BPP, client->buffer[dx*3], client->buffer[dx*3+1], client->buffer[dx*3+2]);
580 CopyRectangle(client, (uint8_t *)&client->buffer[RFB_BUFFER_SIZE / 2], x, y + dy, w, 1);
584 if (!client->jpegError)
585 jpeg_finish_decompress(&cinfo);
587 jpeg_destroy_decompress(&cinfo);
588 free(compressedData);
590 return !client->jpegError;
596 ReadCompactLen (rfbClient* client)
601 if (!ReadFromRFBServer(client, (char *)&b, 1))
605 if (!ReadFromRFBServer(client, (char *)&b, 1))
607 len |= ((int)b & 0x7F) << 7;
609 if (!ReadFromRFBServer(client, (char *)&b, 1))
611 len |= ((int)b & 0xFF) << 14;
618 * JPEG source manager functions for JPEG decompression in Tight decoder.
622 JpegInitSource(j_decompress_ptr cinfo)
624 rfbClient* client=(rfbClient*)cinfo->client_data;
625 client->jpegError = FALSE;
629 JpegFillInputBuffer(j_decompress_ptr cinfo)
631 rfbClient* client=(rfbClient*)cinfo->client_data;
632 client->jpegError = TRUE;
633 client->jpegSrcManager->bytes_in_buffer = client->jpegBufferLen;
634 client->jpegSrcManager->next_input_byte = (JOCTET *)client->jpegBufferPtr;
640 JpegSkipInputData(j_decompress_ptr cinfo, long num_bytes)
642 rfbClient* client=(rfbClient*)cinfo->client_data;
643 if (num_bytes < 0 || num_bytes > client->jpegSrcManager->bytes_in_buffer) {
644 client->jpegError = TRUE;
645 client->jpegSrcManager->bytes_in_buffer = client->jpegBufferLen;
646 client->jpegSrcManager->next_input_byte = (JOCTET *)client->jpegBufferPtr;
648 client->jpegSrcManager->next_input_byte += (size_t) num_bytes;
649 client->jpegSrcManager->bytes_in_buffer -= (size_t) num_bytes;
654 JpegTermSource(j_decompress_ptr cinfo)
656 /* nothing to do here. */
660 JpegSetSrcManager(j_decompress_ptr cinfo,
661 uint8_t *compressedData,
664 rfbClient* client=(rfbClient*)cinfo->client_data;
665 client->jpegBufferPtr = compressedData;
666 client->jpegBufferLen = (size_t)compressedLen;
668 if(client->jpegSrcManager == NULL)
669 client->jpegSrcManager = malloc(sizeof(struct jpeg_source_mgr));
670 client->jpegSrcManager->init_source = JpegInitSource;
671 client->jpegSrcManager->fill_input_buffer = JpegFillInputBuffer;
672 client->jpegSrcManager->skip_input_data = JpegSkipInputData;
673 client->jpegSrcManager->resync_to_restart = jpeg_resync_to_restart;
674 client->jpegSrcManager->term_source = JpegTermSource;
675 client->jpegSrcManager->next_input_byte = (JOCTET*)client->jpegBufferPtr;
676 client->jpegSrcManager->bytes_in_buffer = client->jpegBufferLen;
678 cinfo->src = client->jpegSrcManager;
685 /* LIBVNCSERVER_HAVE_LIBZ and LIBVNCSERVER_HAVE_LIBJPEG */