add libvncserver
[presencevnc] / libvnc / libvncserver / d3des.c
1 /*
2  * This is D3DES (V5.09) by Richard Outerbridge with the double and
3  * triple-length support removed for use in VNC.  Also the bytebit[] array
4  * has been reversed so that the most significant bit in each byte of the
5  * key is ignored, not the least significant.
6  *
7  * These changes are:
8  *  Copyright (C) 1999 AT&T Laboratories Cambridge.  All Rights Reserved.
9  *
10  * This software is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13  */
14
15 /* D3DES (V5.09) -
16  *
17  * A portable, public domain, version of the Data Encryption Standard.
18  *
19  * Written with Symantec's THINK (Lightspeed) C by Richard Outerbridge.
20  * Thanks to: Dan Hoey for his excellent Initial and Inverse permutation
21  * code;  Jim Gillogly & Phil Karn for the DES key schedule code; Dennis
22  * Ferguson, Eric Young and Dana How for comparing notes; and Ray Lau,
23  * for humouring me on.
24  *
25  * Copyright (c) 1988,1989,1990,1991,1992 by Richard Outerbridge.
26  * (GEnie : OUTER; CIS : [71755,204]) Graven Imagery, 1992.
27  */
28
29 #include "d3des.h"
30
31 static void scrunch(unsigned char *, unsigned long *);
32 static void unscrun(unsigned long *, unsigned char *);
33 static void desfunc(unsigned long *, unsigned long *);
34 static void cookey(unsigned long *);
35
36 static unsigned long KnL[32] = { 0L };
37 /*
38 static unsigned long KnR[32] = { 0L };
39 static unsigned long Kn3[32] = { 0L };
40 static unsigned char Df_Key[24] = {
41         0x01,0x23,0x45,0x67,0x89,0xab,0xcd,0xef,
42         0xfe,0xdc,0xba,0x98,0x76,0x54,0x32,0x10,
43         0x89,0xab,0xcd,0xef,0x01,0x23,0x45,0x67 };
44 */
45
46 static unsigned short bytebit[8]        = {
47         01, 02, 04, 010, 020, 040, 0100, 0200 };
48
49 static unsigned long bigbyte[24] = {
50         0x800000L,      0x400000L,      0x200000L,      0x100000L,
51         0x80000L,       0x40000L,       0x20000L,       0x10000L,
52         0x8000L,        0x4000L,        0x2000L,        0x1000L,
53         0x800L,         0x400L,         0x200L,         0x100L,
54         0x80L,          0x40L,          0x20L,          0x10L,
55         0x8L,           0x4L,           0x2L,           0x1L    };
56
57 /* Use the key schedule specified in the Standard (ANSI X3.92-1981). */
58
59 static unsigned char pc1[56] = {
60         56, 48, 40, 32, 24, 16,  8,      0, 57, 49, 41, 33, 25, 17,
61          9,  1, 58, 50, 42, 34, 26,     18, 10,  2, 59, 51, 43, 35,
62         62, 54, 46, 38, 30, 22, 14,      6, 61, 53, 45, 37, 29, 21,
63         13,  5, 60, 52, 44, 36, 28,     20, 12,  4, 27, 19, 11,  3 };
64
65 static unsigned char totrot[16] = {
66         1,2,4,6,8,10,12,14,15,17,19,21,23,25,27,28 };
67
68 static unsigned char pc2[48] = {
69         13, 16, 10, 23,  0,  4,  2, 27, 14,  5, 20,  9,
70         22, 18, 11,  3, 25,  7, 15,  6, 26, 19, 12,  1,
71         40, 51, 30, 36, 46, 54, 29, 39, 50, 44, 32, 47,
72         43, 48, 38, 55, 33, 52, 45, 41, 49, 35, 28, 31 };
73
74 void rfbDesKey(unsigned char *key,
75                int edf)
76 {
77         register int i, j, l, m, n;
78         unsigned char pc1m[56], pcr[56];
79         unsigned long kn[32];
80
81         for ( j = 0; j < 56; j++ ) {
82                 l = pc1[j];
83                 m = l & 07;
84                 pc1m[j] = (key[l >> 3] & bytebit[m]) ? 1 : 0;
85                 }
86         for( i = 0; i < 16; i++ ) {
87                 if( edf == DE1 ) m = (15 - i) << 1;
88                 else m = i << 1;
89                 n = m + 1;
90                 kn[m] = kn[n] = 0L;
91                 for( j = 0; j < 28; j++ ) {
92                         l = j + totrot[i];
93                         if( l < 28 ) pcr[j] = pc1m[l];
94                         else pcr[j] = pc1m[l - 28];
95                         }
96                 for( j = 28; j < 56; j++ ) {
97                     l = j + totrot[i];
98                     if( l < 56 ) pcr[j] = pc1m[l];
99                     else pcr[j] = pc1m[l - 28];
100                     }
101                 for( j = 0; j < 24; j++ ) {
102                         if( pcr[pc2[j]] ) kn[m] |= bigbyte[j];
103                         if( pcr[pc2[j+24]] ) kn[n] |= bigbyte[j];
104                         }
105                 }
106         cookey(kn);
107         return;
108         }
109
110 static void cookey(register unsigned long *raw1)
111 {
112         register unsigned long *cook, *raw0;
113         unsigned long dough[32];
114         register int i;
115
116         cook = dough;
117         for( i = 0; i < 16; i++, raw1++ ) {
118                 raw0 = raw1++;
119                 *cook    = (*raw0 & 0x00fc0000L) << 6;
120                 *cook   |= (*raw0 & 0x00000fc0L) << 10;
121                 *cook   |= (*raw1 & 0x00fc0000L) >> 10;
122                 *cook++ |= (*raw1 & 0x00000fc0L) >> 6;
123                 *cook    = (*raw0 & 0x0003f000L) << 12;
124                 *cook   |= (*raw0 & 0x0000003fL) << 16;
125                 *cook   |= (*raw1 & 0x0003f000L) >> 4;
126                 *cook++ |= (*raw1 & 0x0000003fL);
127                 }
128         rfbUseKey(dough);
129         return;
130         }
131
132 void rfbCPKey(register unsigned long *into)
133 {
134         register unsigned long *from, *endp;
135
136         from = KnL, endp = &KnL[32];
137         while( from < endp ) *into++ = *from++;
138         return;
139         }
140
141 void rfbUseKey(register unsigned long *from)
142 {
143         register unsigned long *to, *endp;
144
145         to = KnL, endp = &KnL[32];
146         while( to < endp ) *to++ = *from++;
147         return;
148         }
149
150 void rfbDes(unsigned char *inblock,
151             unsigned char *outblock)
152 {
153         unsigned long work[2];
154
155         scrunch(inblock, work);
156         desfunc(work, KnL);
157         unscrun(work, outblock);
158         return;
159         }
160
161 static void scrunch(register unsigned char *outof,
162                     register unsigned long *into)
163 {
164         *into    = (*outof++ & 0xffL) << 24;
165         *into   |= (*outof++ & 0xffL) << 16;
166         *into   |= (*outof++ & 0xffL) << 8;
167         *into++ |= (*outof++ & 0xffL);
168         *into    = (*outof++ & 0xffL) << 24;
169         *into   |= (*outof++ & 0xffL) << 16;
170         *into   |= (*outof++ & 0xffL) << 8;
171         *into   |= (*outof   & 0xffL);
172         return;
173         }
174
175 static void unscrun(register unsigned long *outof,
176                     register unsigned char *into)
177 {
178         *into++ = (unsigned char)((*outof >> 24) & 0xffL);
179         *into++ = (unsigned char)((*outof >> 16) & 0xffL);
180         *into++ = (unsigned char)((*outof >>  8) & 0xffL);
181         *into++ = (unsigned char)( *outof++      & 0xffL);
182         *into++ = (unsigned char)((*outof >> 24) & 0xffL);
183         *into++ = (unsigned char)((*outof >> 16) & 0xffL);
184         *into++ = (unsigned char)((*outof >>  8) & 0xffL);
185         *into   = (unsigned char)( *outof        & 0xffL);
186         return;
187         }
188
189 static unsigned long SP1[64] = {
190         0x01010400L, 0x00000000L, 0x00010000L, 0x01010404L,
191         0x01010004L, 0x00010404L, 0x00000004L, 0x00010000L,
192         0x00000400L, 0x01010400L, 0x01010404L, 0x00000400L,
193         0x01000404L, 0x01010004L, 0x01000000L, 0x00000004L,
194         0x00000404L, 0x01000400L, 0x01000400L, 0x00010400L,
195         0x00010400L, 0x01010000L, 0x01010000L, 0x01000404L,
196         0x00010004L, 0x01000004L, 0x01000004L, 0x00010004L,
197         0x00000000L, 0x00000404L, 0x00010404L, 0x01000000L,
198         0x00010000L, 0x01010404L, 0x00000004L, 0x01010000L,
199         0x01010400L, 0x01000000L, 0x01000000L, 0x00000400L,
200         0x01010004L, 0x00010000L, 0x00010400L, 0x01000004L,
201         0x00000400L, 0x00000004L, 0x01000404L, 0x00010404L,
202         0x01010404L, 0x00010004L, 0x01010000L, 0x01000404L,
203         0x01000004L, 0x00000404L, 0x00010404L, 0x01010400L,
204         0x00000404L, 0x01000400L, 0x01000400L, 0x00000000L,
205         0x00010004L, 0x00010400L, 0x00000000L, 0x01010004L };
206
207 static unsigned long SP2[64] = {
208         0x80108020L, 0x80008000L, 0x00008000L, 0x00108020L,
209         0x00100000L, 0x00000020L, 0x80100020L, 0x80008020L,
210         0x80000020L, 0x80108020L, 0x80108000L, 0x80000000L,
211         0x80008000L, 0x00100000L, 0x00000020L, 0x80100020L,
212         0x00108000L, 0x00100020L, 0x80008020L, 0x00000000L,
213         0x80000000L, 0x00008000L, 0x00108020L, 0x80100000L,
214         0x00100020L, 0x80000020L, 0x00000000L, 0x00108000L,
215         0x00008020L, 0x80108000L, 0x80100000L, 0x00008020L,
216         0x00000000L, 0x00108020L, 0x80100020L, 0x00100000L,
217         0x80008020L, 0x80100000L, 0x80108000L, 0x00008000L,
218         0x80100000L, 0x80008000L, 0x00000020L, 0x80108020L,
219         0x00108020L, 0x00000020L, 0x00008000L, 0x80000000L,
220         0x00008020L, 0x80108000L, 0x00100000L, 0x80000020L,
221         0x00100020L, 0x80008020L, 0x80000020L, 0x00100020L,
222         0x00108000L, 0x00000000L, 0x80008000L, 0x00008020L,
223         0x80000000L, 0x80100020L, 0x80108020L, 0x00108000L };
224
225 static unsigned long SP3[64] = {
226         0x00000208L, 0x08020200L, 0x00000000L, 0x08020008L,
227         0x08000200L, 0x00000000L, 0x00020208L, 0x08000200L,
228         0x00020008L, 0x08000008L, 0x08000008L, 0x00020000L,
229         0x08020208L, 0x00020008L, 0x08020000L, 0x00000208L,
230         0x08000000L, 0x00000008L, 0x08020200L, 0x00000200L,
231         0x00020200L, 0x08020000L, 0x08020008L, 0x00020208L,
232         0x08000208L, 0x00020200L, 0x00020000L, 0x08000208L,
233         0x00000008L, 0x08020208L, 0x00000200L, 0x08000000L,
234         0x08020200L, 0x08000000L, 0x00020008L, 0x00000208L,
235         0x00020000L, 0x08020200L, 0x08000200L, 0x00000000L,
236         0x00000200L, 0x00020008L, 0x08020208L, 0x08000200L,
237         0x08000008L, 0x00000200L, 0x00000000L, 0x08020008L,
238         0x08000208L, 0x00020000L, 0x08000000L, 0x08020208L,
239         0x00000008L, 0x00020208L, 0x00020200L, 0x08000008L,
240         0x08020000L, 0x08000208L, 0x00000208L, 0x08020000L,
241         0x00020208L, 0x00000008L, 0x08020008L, 0x00020200L };
242
243 static unsigned long SP4[64] = {
244         0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
245         0x00802080L, 0x00800081L, 0x00800001L, 0x00002001L,
246         0x00000000L, 0x00802000L, 0x00802000L, 0x00802081L,
247         0x00000081L, 0x00000000L, 0x00800080L, 0x00800001L,
248         0x00000001L, 0x00002000L, 0x00800000L, 0x00802001L,
249         0x00000080L, 0x00800000L, 0x00002001L, 0x00002080L,
250         0x00800081L, 0x00000001L, 0x00002080L, 0x00800080L,
251         0x00002000L, 0x00802080L, 0x00802081L, 0x00000081L,
252         0x00800080L, 0x00800001L, 0x00802000L, 0x00802081L,
253         0x00000081L, 0x00000000L, 0x00000000L, 0x00802000L,
254         0x00002080L, 0x00800080L, 0x00800081L, 0x00000001L,
255         0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
256         0x00802081L, 0x00000081L, 0x00000001L, 0x00002000L,
257         0x00800001L, 0x00002001L, 0x00802080L, 0x00800081L,
258         0x00002001L, 0x00002080L, 0x00800000L, 0x00802001L,
259         0x00000080L, 0x00800000L, 0x00002000L, 0x00802080L };
260
261 static unsigned long SP5[64] = {
262         0x00000100L, 0x02080100L, 0x02080000L, 0x42000100L,
263         0x00080000L, 0x00000100L, 0x40000000L, 0x02080000L,
264         0x40080100L, 0x00080000L, 0x02000100L, 0x40080100L,
265         0x42000100L, 0x42080000L, 0x00080100L, 0x40000000L,
266         0x02000000L, 0x40080000L, 0x40080000L, 0x00000000L,
267         0x40000100L, 0x42080100L, 0x42080100L, 0x02000100L,
268         0x42080000L, 0x40000100L, 0x00000000L, 0x42000000L,
269         0x02080100L, 0x02000000L, 0x42000000L, 0x00080100L,
270         0x00080000L, 0x42000100L, 0x00000100L, 0x02000000L,
271         0x40000000L, 0x02080000L, 0x42000100L, 0x40080100L,
272         0x02000100L, 0x40000000L, 0x42080000L, 0x02080100L,
273         0x40080100L, 0x00000100L, 0x02000000L, 0x42080000L,
274         0x42080100L, 0x00080100L, 0x42000000L, 0x42080100L,
275         0x02080000L, 0x00000000L, 0x40080000L, 0x42000000L,
276         0x00080100L, 0x02000100L, 0x40000100L, 0x00080000L,
277         0x00000000L, 0x40080000L, 0x02080100L, 0x40000100L };
278
279 static unsigned long SP6[64] = {
280         0x20000010L, 0x20400000L, 0x00004000L, 0x20404010L,
281         0x20400000L, 0x00000010L, 0x20404010L, 0x00400000L,
282         0x20004000L, 0x00404010L, 0x00400000L, 0x20000010L,
283         0x00400010L, 0x20004000L, 0x20000000L, 0x00004010L,
284         0x00000000L, 0x00400010L, 0x20004010L, 0x00004000L,
285         0x00404000L, 0x20004010L, 0x00000010L, 0x20400010L,
286         0x20400010L, 0x00000000L, 0x00404010L, 0x20404000L,
287         0x00004010L, 0x00404000L, 0x20404000L, 0x20000000L,
288         0x20004000L, 0x00000010L, 0x20400010L, 0x00404000L,
289         0x20404010L, 0x00400000L, 0x00004010L, 0x20000010L,
290         0x00400000L, 0x20004000L, 0x20000000L, 0x00004010L,
291         0x20000010L, 0x20404010L, 0x00404000L, 0x20400000L,
292         0x00404010L, 0x20404000L, 0x00000000L, 0x20400010L,
293         0x00000010L, 0x00004000L, 0x20400000L, 0x00404010L,
294         0x00004000L, 0x00400010L, 0x20004010L, 0x00000000L,
295         0x20404000L, 0x20000000L, 0x00400010L, 0x20004010L };
296
297 static unsigned long SP7[64] = {
298         0x00200000L, 0x04200002L, 0x04000802L, 0x00000000L,
299         0x00000800L, 0x04000802L, 0x00200802L, 0x04200800L,
300         0x04200802L, 0x00200000L, 0x00000000L, 0x04000002L,
301         0x00000002L, 0x04000000L, 0x04200002L, 0x00000802L,
302         0x04000800L, 0x00200802L, 0x00200002L, 0x04000800L,
303         0x04000002L, 0x04200000L, 0x04200800L, 0x00200002L,
304         0x04200000L, 0x00000800L, 0x00000802L, 0x04200802L,
305         0x00200800L, 0x00000002L, 0x04000000L, 0x00200800L,
306         0x04000000L, 0x00200800L, 0x00200000L, 0x04000802L,
307         0x04000802L, 0x04200002L, 0x04200002L, 0x00000002L,
308         0x00200002L, 0x04000000L, 0x04000800L, 0x00200000L,
309         0x04200800L, 0x00000802L, 0x00200802L, 0x04200800L,
310         0x00000802L, 0x04000002L, 0x04200802L, 0x04200000L,
311         0x00200800L, 0x00000000L, 0x00000002L, 0x04200802L,
312         0x00000000L, 0x00200802L, 0x04200000L, 0x00000800L,
313         0x04000002L, 0x04000800L, 0x00000800L, 0x00200002L };
314
315 static unsigned long SP8[64] = {
316         0x10001040L, 0x00001000L, 0x00040000L, 0x10041040L,
317         0x10000000L, 0x10001040L, 0x00000040L, 0x10000000L,
318         0x00040040L, 0x10040000L, 0x10041040L, 0x00041000L,
319         0x10041000L, 0x00041040L, 0x00001000L, 0x00000040L,
320         0x10040000L, 0x10000040L, 0x10001000L, 0x00001040L,
321         0x00041000L, 0x00040040L, 0x10040040L, 0x10041000L,
322         0x00001040L, 0x00000000L, 0x00000000L, 0x10040040L,
323         0x10000040L, 0x10001000L, 0x00041040L, 0x00040000L,
324         0x00041040L, 0x00040000L, 0x10041000L, 0x00001000L,
325         0x00000040L, 0x10040040L, 0x00001000L, 0x00041040L,
326         0x10001000L, 0x00000040L, 0x10000040L, 0x10040000L,
327         0x10040040L, 0x10000000L, 0x00040000L, 0x10001040L,
328         0x00000000L, 0x10041040L, 0x00040040L, 0x10000040L,
329         0x10040000L, 0x10001000L, 0x10001040L, 0x00000000L,
330         0x10041040L, 0x00041000L, 0x00041000L, 0x00001040L,
331         0x00001040L, 0x00040040L, 0x10000000L, 0x10041000L };
332
333 static void desfunc(register unsigned long *block,
334                     register unsigned long *keys)
335 {
336         register unsigned long fval, work, right, leftt;
337         register int round;
338
339         leftt = block[0];
340         right = block[1];
341         work = ((leftt >> 4) ^ right) & 0x0f0f0f0fL;
342         right ^= work;
343         leftt ^= (work << 4);
344         work = ((leftt >> 16) ^ right) & 0x0000ffffL;
345         right ^= work;
346         leftt ^= (work << 16);
347         work = ((right >> 2) ^ leftt) & 0x33333333L;
348         leftt ^= work;
349         right ^= (work << 2);
350         work = ((right >> 8) ^ leftt) & 0x00ff00ffL;
351         leftt ^= work;
352         right ^= (work << 8);
353         right = ((right << 1) | ((right >> 31) & 1L)) & 0xffffffffL;
354         work = (leftt ^ right) & 0xaaaaaaaaL;
355         leftt ^= work;
356         right ^= work;
357         leftt = ((leftt << 1) | ((leftt >> 31) & 1L)) & 0xffffffffL;
358
359         for( round = 0; round < 8; round++ ) {
360                 work  = (right << 28) | (right >> 4);
361                 work ^= *keys++;
362                 fval  = SP7[ work                & 0x3fL];
363                 fval |= SP5[(work >>  8) & 0x3fL];
364                 fval |= SP3[(work >> 16) & 0x3fL];
365                 fval |= SP1[(work >> 24) & 0x3fL];
366                 work  = right ^ *keys++;
367                 fval |= SP8[ work                & 0x3fL];
368                 fval |= SP6[(work >>  8) & 0x3fL];
369                 fval |= SP4[(work >> 16) & 0x3fL];
370                 fval |= SP2[(work >> 24) & 0x3fL];
371                 leftt ^= fval;
372                 work  = (leftt << 28) | (leftt >> 4);
373                 work ^= *keys++;
374                 fval  = SP7[ work                & 0x3fL];
375                 fval |= SP5[(work >>  8) & 0x3fL];
376                 fval |= SP3[(work >> 16) & 0x3fL];
377                 fval |= SP1[(work >> 24) & 0x3fL];
378                 work  = leftt ^ *keys++;
379                 fval |= SP8[ work                & 0x3fL];
380                 fval |= SP6[(work >>  8) & 0x3fL];
381                 fval |= SP4[(work >> 16) & 0x3fL];
382                 fval |= SP2[(work >> 24) & 0x3fL];
383                 right ^= fval;
384                 }
385
386         right = (right << 31) | (right >> 1);
387         work = (leftt ^ right) & 0xaaaaaaaaL;
388         leftt ^= work;
389         right ^= work;
390         leftt = (leftt << 31) | (leftt >> 1);
391         work = ((leftt >> 8) ^ right) & 0x00ff00ffL;
392         right ^= work;
393         leftt ^= (work << 8);
394         work = ((leftt >> 2) ^ right) & 0x33333333L;
395         right ^= work;
396         leftt ^= (work << 2);
397         work = ((right >> 16) ^ leftt) & 0x0000ffffL;
398         leftt ^= work;
399         right ^= (work << 16);
400         work = ((right >> 4) ^ leftt) & 0x0f0f0f0fL;
401         leftt ^= work;
402         right ^= (work << 4);
403         *block++ = right;
404         *block = leftt;
405         return;
406         }
407
408 /* Validation sets:
409  *
410  * Single-length key, single-length plaintext -
411  * Key    : 0123 4567 89ab cdef
412  * Plain  : 0123 4567 89ab cde7
413  * Cipher : c957 4425 6a5e d31d
414  *
415  * Double-length key, single-length plaintext -
416  * Key    : 0123 4567 89ab cdef fedc ba98 7654 3210
417  * Plain  : 0123 4567 89ab cde7
418  * Cipher : 7f1d 0a77 826b 8aff
419  *
420  * Double-length key, double-length plaintext -
421  * Key    : 0123 4567 89ab cdef fedc ba98 7654 3210
422  * Plain  : 0123 4567 89ab cdef 0123 4567 89ab cdff
423  * Cipher : 27a0 8440 406a df60 278f 47cf 42d6 15d7
424  *
425  * Triple-length key, single-length plaintext -
426  * Key    : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
427  * Plain  : 0123 4567 89ab cde7
428  * Cipher : de0b 7c06 ae5e 0ed5
429  *
430  * Triple-length key, double-length plaintext -
431  * Key    : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
432  * Plain  : 0123 4567 89ab cdef 0123 4567 89ab cdff
433  * Cipher : ad0d 1b30 ac17 cf07 0ed1 1c63 81e4 4de5
434  *
435  * d3des V5.0a rwo 9208.07 18:44 Graven Imagery
436  **********************************************************************/