fix disconnect bug in libvncserver
[presencevnc] / libvnc / libvncserver / translate.c
1 /*
2  * translate.c - translate between different pixel formats
3  */
4
5 /*
6  *  OSXvnc Copyright (C) 2001 Dan McGuirk <mcguirk@incompleteness.net>.
7  *  Original Xvnc code Copyright (C) 1999 AT&T Laboratories Cambridge.  
8  *  All Rights Reserved.
9  *
10  *  This is free software; you can redistribute it and/or modify
11  *  it under the terms of the GNU General Public License as published by
12  *  the Free Software Foundation; either version 2 of the License, or
13  *  (at your option) any later version.
14  *
15  *  This software is distributed in the hope that it will be useful,
16  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
17  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18  *  GNU General Public License for more details.
19  *
20  *  You should have received a copy of the GNU General Public License
21  *  along with this software; if not, write to the Free Software
22  *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
23  *  USA.
24  */
25
26 #include <rfb/rfb.h>
27 #include <rfb/rfbregion.h>
28
29 static void PrintPixelFormat(rfbPixelFormat *pf);
30 static rfbBool rfbSetClientColourMapBGR233(rfbClientPtr cl);
31
32 rfbBool rfbEconomicTranslate = FALSE;
33
34 /*
35  * Some standard pixel formats.
36  */
37
38 static const rfbPixelFormat BGR233Format = {
39     8, 8, 0, 1, 7, 7, 3, 0, 3, 6, 0, 0
40 };
41
42
43 /*
44  * Macro to compare pixel formats.
45  */
46
47 #define PF_EQ(x,y)                                                      \
48         ((x.bitsPerPixel == y.bitsPerPixel) &&                          \
49          (x.depth == y.depth) &&                                        \
50          ((x.bigEndian == y.bigEndian) || (x.bitsPerPixel == 8)) &&     \
51          (x.trueColour == y.trueColour) &&                              \
52          (!x.trueColour || ((x.redMax == y.redMax) &&                   \
53                             (x.greenMax == y.greenMax) &&               \
54                             (x.blueMax == y.blueMax) &&                 \
55                             (x.redShift == y.redShift) &&               \
56                             (x.greenShift == y.greenShift) &&           \
57                             (x.blueShift == y.blueShift))))
58
59 #define CONCAT2(a,b) a##b
60 #define CONCAT2E(a,b) CONCAT2(a,b)
61 #define CONCAT3(a,b,c) a##b##c
62 #define CONCAT3E(a,b,c) CONCAT3(a,b,c)
63 #define CONCAT4(a,b,c,d) a##b##c##d
64 #define CONCAT4E(a,b,c,d) CONCAT4(a,b,c,d)
65
66 #undef OUT
67 #undef IN
68
69 #define OUT 8
70 #include "tableinitcmtemplate.c"
71 #include "tableinittctemplate.c"
72 #define IN 8
73 #include "tabletranstemplate.c"
74 #undef IN
75 #define IN 16
76 #include "tabletranstemplate.c"
77 #undef IN
78 #define IN 32
79 #include "tabletranstemplate.c"
80 #undef IN
81 #undef OUT
82
83 #define OUT 16
84 #include "tableinitcmtemplate.c"
85 #include "tableinittctemplate.c"
86 #define IN 8
87 #include "tabletranstemplate.c"
88 #undef IN
89 #define IN 16
90 #include "tabletranstemplate.c"
91 #undef IN
92 #define IN 32
93 #include "tabletranstemplate.c"
94 #undef IN
95 #undef OUT
96
97 #define OUT 32
98 #include "tableinitcmtemplate.c"
99 #include "tableinittctemplate.c"
100 #define IN 8
101 #include "tabletranstemplate.c"
102 #undef IN
103 #define IN 16
104 #include "tabletranstemplate.c"
105 #undef IN
106 #define IN 32
107 #include "tabletranstemplate.c"
108 #undef IN
109 #undef OUT
110
111 #ifdef LIBVNCSERVER_ALLOW24BPP
112 #define COUNT_OFFSETS 4
113 #define BPP2OFFSET(bpp) ((bpp)/8-1)
114 #include "tableinit24.c"
115 #define BPP 8
116 #include "tabletrans24template.c"
117 #undef BPP
118 #define BPP 16
119 #include "tabletrans24template.c"
120 #undef BPP
121 #define BPP 24
122 #include "tabletrans24template.c"
123 #undef BPP
124 #define BPP 32
125 #include "tabletrans24template.c"
126 #undef BPP
127 #else
128 #define COUNT_OFFSETS 3
129 #define BPP2OFFSET(bpp) ((int)(bpp)/16)
130 #endif
131
132 typedef void (*rfbInitCMTableFnType)(char **table, rfbPixelFormat *in,
133                                    rfbPixelFormat *out,rfbColourMap* cm);
134 typedef void (*rfbInitTableFnType)(char **table, rfbPixelFormat *in,
135                                    rfbPixelFormat *out);
136
137 static rfbInitCMTableFnType rfbInitColourMapSingleTableFns[COUNT_OFFSETS] = {
138     rfbInitColourMapSingleTable8,
139     rfbInitColourMapSingleTable16,
140 #ifdef LIBVNCSERVER_ALLOW24BPP
141     rfbInitColourMapSingleTable24,
142 #endif
143     rfbInitColourMapSingleTable32
144 };
145
146 static rfbInitTableFnType rfbInitTrueColourSingleTableFns[COUNT_OFFSETS] = {
147     rfbInitTrueColourSingleTable8,
148     rfbInitTrueColourSingleTable16,
149 #ifdef LIBVNCSERVER_ALLOW24BPP
150     rfbInitTrueColourSingleTable24,
151 #endif
152     rfbInitTrueColourSingleTable32
153 };
154
155 static rfbInitTableFnType rfbInitTrueColourRGBTablesFns[COUNT_OFFSETS] = {
156     rfbInitTrueColourRGBTables8,
157     rfbInitTrueColourRGBTables16,
158 #ifdef LIBVNCSERVER_ALLOW24BPP
159     rfbInitTrueColourRGBTables24,
160 #endif
161     rfbInitTrueColourRGBTables32
162 };
163
164 static rfbTranslateFnType rfbTranslateWithSingleTableFns[COUNT_OFFSETS][COUNT_OFFSETS] = {
165     { rfbTranslateWithSingleTable8to8,
166       rfbTranslateWithSingleTable8to16,
167 #ifdef LIBVNCSERVER_ALLOW24BPP
168       rfbTranslateWithSingleTable8to24,
169 #endif
170       rfbTranslateWithSingleTable8to32 },
171     { rfbTranslateWithSingleTable16to8,
172       rfbTranslateWithSingleTable16to16,
173 #ifdef LIBVNCSERVER_ALLOW24BPP
174       rfbTranslateWithSingleTable16to24,
175 #endif
176       rfbTranslateWithSingleTable16to32 },
177 #ifdef LIBVNCSERVER_ALLOW24BPP
178     { rfbTranslateWithSingleTable24to8,
179       rfbTranslateWithSingleTable24to16,
180       rfbTranslateWithSingleTable24to24,
181       rfbTranslateWithSingleTable24to32 },
182 #endif
183     { rfbTranslateWithSingleTable32to8,
184       rfbTranslateWithSingleTable32to16,
185 #ifdef LIBVNCSERVER_ALLOW24BPP
186       rfbTranslateWithSingleTable32to24,
187 #endif
188       rfbTranslateWithSingleTable32to32 }
189 };
190
191 static rfbTranslateFnType rfbTranslateWithRGBTablesFns[COUNT_OFFSETS][COUNT_OFFSETS] = {
192     { rfbTranslateWithRGBTables8to8,
193       rfbTranslateWithRGBTables8to16,
194 #ifdef LIBVNCSERVER_ALLOW24BPP
195       rfbTranslateWithRGBTables8to24,
196 #endif
197       rfbTranslateWithRGBTables8to32 },
198     { rfbTranslateWithRGBTables16to8,
199       rfbTranslateWithRGBTables16to16,
200 #ifdef LIBVNCSERVER_ALLOW24BPP
201       rfbTranslateWithRGBTables16to24,
202 #endif
203       rfbTranslateWithRGBTables16to32 },
204 #ifdef LIBVNCSERVER_ALLOW24BPP
205     { rfbTranslateWithRGBTables24to8,
206       rfbTranslateWithRGBTables24to16,
207       rfbTranslateWithRGBTables24to24,
208       rfbTranslateWithRGBTables24to32 },
209 #endif
210     { rfbTranslateWithRGBTables32to8,
211       rfbTranslateWithRGBTables32to16,
212 #ifdef LIBVNCSERVER_ALLOW24BPP
213       rfbTranslateWithRGBTables32to24,
214 #endif
215       rfbTranslateWithRGBTables32to32 }
216 };
217
218
219
220 /*
221  * rfbTranslateNone is used when no translation is required.
222  */
223
224 void
225 rfbTranslateNone(char *table, rfbPixelFormat *in, rfbPixelFormat *out,
226                  char *iptr, char *optr, int bytesBetweenInputLines,
227                  int width, int height)
228 {
229     int bytesPerOutputLine = width * (out->bitsPerPixel / 8);
230
231     while (height > 0) {
232         memcpy(optr, iptr, bytesPerOutputLine);
233         iptr += bytesBetweenInputLines;
234         optr += bytesPerOutputLine;
235         height--;
236     }
237 }
238
239
240 /*
241  * rfbSetTranslateFunction sets the translation function.
242  */
243
244 rfbBool
245 rfbSetTranslateFunction(rfbClientPtr cl)
246 {
247     rfbLog("Pixel format for client %s:\n",cl->host);
248     PrintPixelFormat(&cl->format);
249
250     /*
251      * Check that bits per pixel values are valid
252      */
253
254     if ((cl->screen->serverFormat.bitsPerPixel != 8) &&
255         (cl->screen->serverFormat.bitsPerPixel != 16) &&
256 #ifdef LIBVNCSERVER_ALLOW24BPP
257         (cl->screen->serverFormat.bitsPerPixel != 24) &&
258 #endif
259         (cl->screen->serverFormat.bitsPerPixel != 32))
260     {
261         rfbErr("%s: server bits per pixel not 8, 16 or 32 (is %d)\n",
262                "rfbSetTranslateFunction", 
263                cl->screen->serverFormat.bitsPerPixel);
264         rfbCloseClient(cl);
265         return FALSE;
266     }
267
268     if ((cl->format.bitsPerPixel != 8) &&
269         (cl->format.bitsPerPixel != 16) &&
270 #ifdef LIBVNCSERVER_ALLOW24BPP
271         (cl->format.bitsPerPixel != 24) &&
272 #endif
273         (cl->format.bitsPerPixel != 32))
274     {
275         rfbErr("%s: client bits per pixel not 8, 16 or 32\n",
276                 "rfbSetTranslateFunction");
277         rfbCloseClient(cl);
278         return FALSE;
279     }
280
281     if (!cl->format.trueColour && (cl->format.bitsPerPixel != 8)) {
282         rfbErr("rfbSetTranslateFunction: client has colour map "
283                 "but %d-bit - can only cope with 8-bit colour maps\n",
284                 cl->format.bitsPerPixel);
285         rfbCloseClient(cl);
286         return FALSE;
287     }
288
289     /*
290      * bpp is valid, now work out how to translate
291      */
292
293     if (!cl->format.trueColour) {
294         /*
295          * truecolour -> colour map
296          *
297          * Set client's colour map to BGR233, then effectively it's
298          * truecolour as well
299          */
300
301         if (!rfbSetClientColourMapBGR233(cl))
302             return FALSE;
303
304         cl->format = BGR233Format;
305     }
306
307     /* truecolour -> truecolour */
308
309     if (PF_EQ(cl->format,cl->screen->serverFormat)) {
310
311         /* client & server the same */
312
313         rfbLog("no translation needed\n");
314         cl->translateFn = rfbTranslateNone;
315         return TRUE;
316     }
317
318     if ((cl->screen->serverFormat.bitsPerPixel < 16) ||
319         ((!cl->screen->serverFormat.trueColour || !rfbEconomicTranslate) &&
320            (cl->screen->serverFormat.bitsPerPixel == 16))) {
321
322         /* we can use a single lookup table for <= 16 bpp */
323
324         cl->translateFn = rfbTranslateWithSingleTableFns
325                               [BPP2OFFSET(cl->screen->serverFormat.bitsPerPixel)]
326                                   [BPP2OFFSET(cl->format.bitsPerPixel)];
327
328         if(cl->screen->serverFormat.trueColour)
329           (*rfbInitTrueColourSingleTableFns
330            [BPP2OFFSET(cl->format.bitsPerPixel)]) (&cl->translateLookupTable,
331                                                    &(cl->screen->serverFormat), &cl->format);
332         else
333           (*rfbInitColourMapSingleTableFns
334            [BPP2OFFSET(cl->format.bitsPerPixel)]) (&cl->translateLookupTable,
335                                                    &(cl->screen->serverFormat), &cl->format,&cl->screen->colourMap);
336
337     } else {
338
339         /* otherwise we use three separate tables for red, green and blue */
340
341         cl->translateFn = rfbTranslateWithRGBTablesFns
342                               [BPP2OFFSET(cl->screen->serverFormat.bitsPerPixel)]
343                                   [BPP2OFFSET(cl->format.bitsPerPixel)];
344
345         (*rfbInitTrueColourRGBTablesFns
346             [BPP2OFFSET(cl->format.bitsPerPixel)]) (&cl->translateLookupTable,
347                                              &(cl->screen->serverFormat), &cl->format);
348     }
349
350     return TRUE;
351 }
352
353
354
355 /*
356  * rfbSetClientColourMapBGR233 sets the client's colour map so that it's
357  * just like an 8-bit BGR233 true colour client.
358  */
359
360 static rfbBool
361 rfbSetClientColourMapBGR233(rfbClientPtr cl)
362 {
363     char buf[sz_rfbSetColourMapEntriesMsg + 256 * 3 * 2];
364     rfbSetColourMapEntriesMsg *scme = (rfbSetColourMapEntriesMsg *)buf;
365     uint16_t *rgb = (uint16_t *)(&buf[sz_rfbSetColourMapEntriesMsg]);
366     int i, len;
367     int r, g, b;
368
369     if (cl->format.bitsPerPixel != 8 ) {
370         rfbErr("%s: client not 8 bits per pixel\n",
371                 "rfbSetClientColourMapBGR233");
372         rfbCloseClient(cl);
373         return FALSE;
374     }
375     
376     scme->type = rfbSetColourMapEntries;
377
378     scme->firstColour = Swap16IfLE(0);
379     scme->nColours = Swap16IfLE(256);
380
381     len = sz_rfbSetColourMapEntriesMsg;
382
383     i = 0;
384
385     for (b = 0; b < 4; b++) {
386         for (g = 0; g < 8; g++) {
387             for (r = 0; r < 8; r++) {
388                 rgb[i++] = Swap16IfLE(r * 65535 / 7);
389                 rgb[i++] = Swap16IfLE(g * 65535 / 7);
390                 rgb[i++] = Swap16IfLE(b * 65535 / 3);
391             }
392         }
393     }
394
395     len += 256 * 3 * 2;
396
397     if (rfbWriteExact(cl, buf, len) < 0) {
398         rfbLogPerror("rfbSetClientColourMapBGR233: write");
399         rfbCloseClient(cl);
400         return FALSE;
401     }
402     return TRUE;
403 }
404
405 /* this function is not called very often, so it needn't be
406    efficient. */
407
408 /*
409  * rfbSetClientColourMap is called to set the client's colour map.  If the
410  * client is a true colour client, we simply update our own translation table
411  * and mark the whole screen as having been modified.
412  */
413
414 rfbBool
415 rfbSetClientColourMap(rfbClientPtr cl, int firstColour, int nColours)
416 {
417     if (cl->screen->serverFormat.trueColour || !cl->readyForSetColourMapEntries) {
418         return TRUE;
419     }
420
421     if (nColours == 0) {
422         nColours = cl->screen->colourMap.count;
423     }
424
425     if (cl->format.trueColour) {
426         (*rfbInitColourMapSingleTableFns
427             [BPP2OFFSET(cl->format.bitsPerPixel)]) (&cl->translateLookupTable,
428                                              &cl->screen->serverFormat, &cl->format,&cl->screen->colourMap);
429
430         sraRgnDestroy(cl->modifiedRegion);
431         cl->modifiedRegion =
432           sraRgnCreateRect(0,0,cl->screen->width,cl->screen->height);
433
434         return TRUE;
435     }
436
437     return rfbSendSetColourMapEntries(cl, firstColour, nColours);
438 }
439
440
441 /*
442  * rfbSetClientColourMaps sets the colour map for each RFB client.
443  */
444
445 void
446 rfbSetClientColourMaps(rfbScreenInfoPtr rfbScreen, int firstColour, int nColours)
447 {
448     rfbClientIteratorPtr i;
449     rfbClientPtr cl;
450
451     i = rfbGetClientIterator(rfbScreen);
452     while((cl = rfbClientIteratorNext(i)))
453       rfbSetClientColourMap(cl, firstColour, nColours);
454     rfbReleaseClientIterator(i);
455 }
456
457 static void
458 PrintPixelFormat(rfbPixelFormat *pf)
459 {
460     if (pf->bitsPerPixel == 1) {
461         rfbLog("  1 bpp, %s sig bit in each byte is leftmost on the screen.\n",
462                (pf->bigEndian ? "most" : "least"));
463     } else {
464         rfbLog("  %d bpp, depth %d%s\n",pf->bitsPerPixel,pf->depth,
465                ((pf->bitsPerPixel == 8) ? ""
466                 : (pf->bigEndian ? ", big endian" : ", little endian")));
467         if (pf->trueColour) {
468             rfbLog("  true colour: max r %d g %d b %d, shift r %d g %d b %d\n",
469                    pf->redMax, pf->greenMax, pf->blueMax,
470                    pf->redShift, pf->greenShift, pf->blueShift);
471         } else {
472             rfbLog("  uses a colour map (not true colour).\n");
473         }
474     }
475 }