LibVNCServer/LibVNCClient
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
zrle.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2005 Johannes E. Schindelin. All Rights Reserved.
3  *
4  * This is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation; either version 2 of the License, or
7  * (at your option) any later version.
8  *
9  * This software is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this software; if not, write to the Free Software
16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
17  * USA.
18  */
19 
20 #ifdef LIBVNCSERVER_HAVE_LIBZ
21 
22 /*
23  * zrle.c - handle zrle encoding.
24  *
25  * This file shouldn't be compiled directly. It is included multiple times by
26  * rfbproto.c, each time with a different definition of the macro BPP. For
27  * each value of BPP, this file defines a function which handles an zrle
28  * encoded rectangle with BPP bits per pixel.
29  */
30 
31 #ifndef REALBPP
32 #define REALBPP BPP
33 #endif
34 
35 #if !defined(UNCOMP) || UNCOMP==0
36 #define HandleZRLE CONCAT2E(HandleZRLE,REALBPP)
37 #define HandleZRLETile CONCAT2E(HandleZRLETile,REALBPP)
38 #elif UNCOMP>0
39 #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Down)
40 #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Down)
41 #else
42 #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Up)
43 #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Up)
44 #endif
45 #define CARDBPP CONCAT3E(uint,BPP,_t)
46 #define CARDREALBPP CONCAT3E(uint,REALBPP,_t)
47 
48 #define ENDIAN_LITTLE 0
49 #define ENDIAN_BIG 1
50 #define ENDIAN_NO 2
51 #define ZYWRLE_ENDIAN ENDIAN_LITTLE
52 #undef END_FIX
53 #if ZYWRLE_ENDIAN == ENDIAN_LITTLE
54 # define END_FIX LE
55 #elif ZYWRLE_ENDIAN == ENDIAN_BIG
56 # define END_FIX BE
57 #else
58 # define END_FIX NE
59 #endif
60 #define __RFB_CONCAT3E(a,b,c) CONCAT3E(a,b,c)
61 #define __RFB_CONCAT2E(a,b) CONCAT2E(a,b)
62 #undef CPIXEL
63 #if REALBPP != BPP
64 #if UNCOMP == 0
65 #define CPIXEL REALBPP
66 #elif UNCOMP>0
67 #define CPIXEL CONCAT2E(REALBPP,Down)
68 #else
69 #define CPIXEL CONCAT2E(REALBPP,Up)
70 #endif
71 #endif
72 #define PIXEL_T __RFB_CONCAT3E(uint,BPP,_t)
73 #if BPP!=8
74 #define ZYWRLE_DECODE 1
75 #include "zywrletemplate.c"
76 #endif
77 #undef CPIXEL
78 
79 static int HandleZRLETile(rfbClient* client,
80  uint8_t* buffer,size_t buffer_length,
81  int x,int y,int w,int h);
82 
83 static rfbBool
84 HandleZRLE (rfbClient* client, int rx, int ry, int rw, int rh)
85 {
86  rfbZRLEHeader header;
87  int remaining;
88  int inflateResult;
89  int toRead;
90  int min_buffer_size = rw * rh * (REALBPP / 8) * 2;
91 
92  /* First make sure we have a large enough raw buffer to hold the
93  * decompressed data. In practice, with a fixed REALBPP, fixed frame
94  * buffer size and the first update containing the entire frame
95  * buffer, this buffer allocation should only happen once, on the
96  * first update.
97  */
98  if ( client->raw_buffer_size < min_buffer_size) {
99 
100  if ( client->raw_buffer != NULL ) {
101 
102  free( client->raw_buffer );
103 
104  }
105 
106  client->raw_buffer_size = min_buffer_size;
107  client->raw_buffer = (char*) malloc( client->raw_buffer_size );
108 
109  }
110 
111  if (!ReadFromRFBServer(client, (char *)&header, sz_rfbZRLEHeader))
112  return FALSE;
113 
114  remaining = rfbClientSwap32IfLE(header.length);
115 
116  /* Need to initialize the decompressor state. */
117  client->decompStream.next_in = ( Bytef * )client->buffer;
118  client->decompStream.avail_in = 0;
119  client->decompStream.next_out = ( Bytef * )client->raw_buffer;
120  client->decompStream.avail_out = client->raw_buffer_size;
121  client->decompStream.data_type = Z_BINARY;
122 
123  /* Initialize the decompression stream structures on the first invocation. */
124  if ( client->decompStreamInited == FALSE ) {
125 
126  inflateResult = inflateInit( &client->decompStream );
127 
128  if ( inflateResult != Z_OK ) {
129  rfbClientLog(
130  "inflateInit returned error: %d, msg: %s\n",
131  inflateResult,
132  client->decompStream.msg);
133  return FALSE;
134  }
135 
136  client->decompStreamInited = TRUE;
137 
138  }
139 
140  inflateResult = Z_OK;
141 
142  /* Process buffer full of data until no more to process, or
143  * some type of inflater error, or Z_STREAM_END.
144  */
145  while (( remaining > 0 ) &&
146  ( inflateResult == Z_OK )) {
147 
148  if ( remaining > RFB_BUFFER_SIZE ) {
149  toRead = RFB_BUFFER_SIZE;
150  }
151  else {
152  toRead = remaining;
153  }
154 
155  /* Fill the buffer, obtaining data from the server. */
156  if (!ReadFromRFBServer(client, client->buffer,toRead))
157  return FALSE;
158 
159  client->decompStream.next_in = ( Bytef * )client->buffer;
160  client->decompStream.avail_in = toRead;
161 
162  /* Need to uncompress buffer full. */
163  inflateResult = inflate( &client->decompStream, Z_SYNC_FLUSH );
164 
165  /* We never supply a dictionary for compression. */
166  if ( inflateResult == Z_NEED_DICT ) {
167  rfbClientLog("zlib inflate needs a dictionary!\n");
168  return FALSE;
169  }
170  if ( inflateResult < 0 ) {
171  rfbClientLog(
172  "zlib inflate returned error: %d, msg: %s\n",
173  inflateResult,
174  client->decompStream.msg);
175  return FALSE;
176  }
177 
178  /* Result buffer allocated to be at least large enough. We should
179  * never run out of space!
180  */
181  if (( client->decompStream.avail_in > 0 ) &&
182  ( client->decompStream.avail_out <= 0 )) {
183  rfbClientLog("zlib inflate ran out of space!\n");
184  return FALSE;
185  }
186 
187  remaining -= toRead;
188 
189  } /* while ( remaining > 0 ) */
190 
191  if ( inflateResult == Z_OK ) {
192  void* buf=client->raw_buffer;
193  int i,j;
194 
195  remaining = client->raw_buffer_size-client->decompStream.avail_out;
196 
197  for(j=0; j<rh; j+=rfbZRLETileHeight)
198  for(i=0; i<rw; i+=rfbZRLETileWidth) {
199  int subWidth=(i+rfbZRLETileWidth>rw)?rw-i:rfbZRLETileWidth;
200  int subHeight=(j+rfbZRLETileHeight>rh)?rh-j:rfbZRLETileHeight;
201  int result=HandleZRLETile(client,buf,remaining,rx+i,ry+j,subWidth,subHeight);
202 
203  if(result<0) {
204  rfbClientLog("ZRLE decoding failed (%d)\n",result);
205 return TRUE;
206  return FALSE;
207  }
208 
209  buf+=result;
210  remaining-=result;
211  }
212  }
213  else {
214 
215  rfbClientLog(
216  "zlib inflate returned error: %d, msg: %s\n",
217  inflateResult,
218  client->decompStream.msg);
219  return FALSE;
220 
221  }
222 
223  return TRUE;
224 }
225 
226 #if REALBPP!=BPP && defined(UNCOMP) && UNCOMP!=0
227 #if UNCOMP>0
228 #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)>>UNCOMP)
229 #else
230 #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)<<(-(UNCOMP)))
231 #endif
232 #else
233 #define UncompressCPixel(pointer) (*(CARDBPP*)pointer)
234 #endif
235 
236 static int HandleZRLETile(rfbClient* client,
237  uint8_t* buffer,size_t buffer_length,
238  int x,int y,int w,int h) {
239  uint8_t* buffer_copy = buffer;
240  uint8_t* buffer_end = buffer+buffer_length;
241  uint8_t type;
242 #if BPP!=8
243  uint8_t zywrle_level = (client->appData.qualityLevel & 0x80) ?
244  0 : (3 - client->appData.qualityLevel / 3);
245 #endif
246 
247  if(buffer_length<1)
248  return -2;
249 
250  type = *buffer;
251  buffer++;
252  {
253  if( type == 0 ) /* raw */
254 #if BPP!=8
255  if( zywrle_level > 0 ){
256  CARDBPP* pFrame = (CARDBPP*)client->frameBuffer + y*client->width+x;
257  int ret;
258  client->appData.qualityLevel |= 0x80;
259  ret = HandleZRLETile(client, buffer, buffer_end-buffer, x, y, w, h);
260  client->appData.qualityLevel &= 0x7F;
261  if( ret < 0 ){
262  return ret;
263  }
264  ZYWRLE_SYNTHESIZE( pFrame, pFrame, w, h, client->width, zywrle_level, (int*)client->zlib_buffer );
265  buffer += ret;
266  }else
267 #endif
268  {
269 #if REALBPP!=BPP
270  int i,j;
271 
272  if(1+w*h*REALBPP/8>buffer_length) {
273  rfbClientLog("expected %d bytes, got only %d (%dx%d)\n",1+w*h*REALBPP/8,buffer_length,w,h);
274  return -3;
275  }
276 
277  for(j=y*client->width; j<(y+h)*client->width; j+=client->width)
278  for(i=x; i<x+w; i++,buffer+=REALBPP/8)
279  ((CARDBPP*)client->frameBuffer)[j+i] = UncompressCPixel(buffer);
280 #else
281  CopyRectangle(client, buffer, x, y, w, h);
282  buffer+=w*h*REALBPP/8;
283 #endif
284  }
285  else if( type == 1 ) /* solid */
286  {
287  CARDBPP color = UncompressCPixel(buffer);
288 
289  if(1+REALBPP/8>buffer_length)
290  return -4;
291 
292  FillRectangle(client, x, y, w, h, color);
293 
294  buffer+=REALBPP/8;
295 
296  }
297  else if( (type >= 2)&&(type <= 127) ) /* packed Palette */
298  {
299  CARDBPP palette[16];
300  int i,j,shift,
301  bpp=(type>4?(type>16?8:4):(type>2?2:1)),
302  mask=(1<<bpp)-1,
303  divider=(8/bpp);
304 
305  if(1+type*REALBPP/8+((w+divider-1)/divider)*h>buffer_length)
306  return -5;
307 
308  /* read palette */
309  for(i=0; i<type; i++,buffer+=REALBPP/8)
310  palette[i] = UncompressCPixel(buffer);
311 
312  /* read palettized pixels */
313  for(j=y*client->width; j<(y+h)*client->width; j+=client->width) {
314  for(i=x,shift=8-bpp; i<x+w; i++) {
315  ((CARDBPP*)client->frameBuffer)[j+i] = palette[((*buffer)>>shift)&mask];
316  shift-=bpp;
317  if(shift<0) {
318  shift=8-bpp;
319  buffer++;
320  }
321  }
322  if(shift<8-bpp)
323  buffer++;
324  }
325 
326  }
327  /* case 17 ... 127: not used, but valid */
328  else if( type == 128 ) /* plain RLE */
329  {
330  int i=0,j=0;
331  while(j<h) {
332  int color,length;
333  /* read color */
334  if(buffer+REALBPP/8+1>buffer_end)
335  return -7;
336  color = UncompressCPixel(buffer);
337  buffer+=REALBPP/8;
338  /* read run length */
339  length=1;
340  while(*buffer==0xff) {
341  if(buffer+1>=buffer_end)
342  return -8;
343  length+=*buffer;
344  buffer++;
345  }
346  length+=*buffer;
347  buffer++;
348  while(j<h && length>0) {
349  ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color;
350  length--;
351  i++;
352  if(i>=w) {
353  i=0;
354  j++;
355  }
356  }
357  if(length>0)
358  rfbClientLog("Warning: possible ZRLE corruption\n");
359  }
360 
361  }
362  else if( type == 129 ) /* unused */
363  {
364  return -8;
365  }
366  else if( type >= 130 ) /* palette RLE */
367  {
368  CARDBPP palette[128];
369  int i,j;
370 
371  if(2+(type-128)*REALBPP/8>buffer_length)
372  return -9;
373 
374  /* read palette */
375  for(i=0; i<type-128; i++,buffer+=REALBPP/8)
376  palette[i] = UncompressCPixel(buffer);
377  /* read palettized pixels */
378  i=j=0;
379  while(j<h) {
380  int color,length;
381  /* read color */
382  if(buffer>=buffer_end)
383  return -10;
384  color = palette[(*buffer)&0x7f];
385  length=1;
386  if(*buffer&0x80) {
387  if(buffer+1>=buffer_end)
388  return -11;
389  buffer++;
390  /* read run length */
391  while(*buffer==0xff) {
392  if(buffer+1>=buffer_end)
393  return -8;
394  length+=*buffer;
395  buffer++;
396  }
397  length+=*buffer;
398  }
399  buffer++;
400  while(j<h && length>0) {
401  ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color;
402  length--;
403  i++;
404  if(i>=w) {
405  i=0;
406  j++;
407  }
408  }
409  if(length>0)
410  rfbClientLog("Warning: possible ZRLE corruption\n");
411  }
412  }
413  }
414 
415  return buffer-buffer_copy;
416 }
417 
418 #undef CARDBPP
419 #undef CARDREALBPP
420 #undef HandleZRLE
421 #undef HandleZRLETile
422 #undef UncompressCPixel
423 #undef REALBPP
424 
425 #endif
426 
427 #undef UNCOMP