LibVNCServer/LibVNCClient
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
tight.c
Go to the documentation of this file.
1 /*
2  * tight.c
3  *
4  * Routines to implement Tight Encoding
5  *
6  * Our Tight encoder is based roughly on the TurboVNC v0.6 encoder with some
7  * additional enhancements from TurboVNC 1.1. For lower compression levels,
8  * this encoder provides a tremendous reduction in CPU usage (and subsequently,
9  * an increase in throughput for CPU-limited environments) relative to the
10  * TightVNC encoder, whereas Compression Level 9 provides a low-bandwidth mode
11  * that behaves similarly to Compression Levels 5-9 in the old TightVNC
12  * encoder.
13  */
14 
15 /*
16  * Copyright (C) 2010-2012 D. R. Commander. All Rights Reserved.
17  * Copyright (C) 2005-2008 Sun Microsystems, Inc. All Rights Reserved.
18  * Copyright (C) 2004 Landmark Graphics Corporation. All Rights Reserved.
19  * Copyright (C) 2000, 2001 Const Kaplinsky. All Rights Reserved.
20  * Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved.
21  *
22  * This is free software; you can redistribute it and/or modify
23  * it under the terms of the GNU General Public License as published by
24  * the Free Software Foundation; either version 2 of the License, or
25  * (at your option) any later version.
26  *
27  * This software is distributed in the hope that it will be useful,
28  * but WITHOUT ANY WARRANTY; without even the implied warranty of
29  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
30  * GNU General Public License for more details.
31  *
32  * You should have received a copy of the GNU General Public License
33  * along with this software; if not, write to the Free Software
34  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
35  * USA.
36  */
37 
38 #include <rfb/rfb.h>
39 #include "private.h"
40 
41 #ifdef LIBVNCSERVER_HAVE_LIBPNG
42 #include <png.h>
43 #endif
44 #include "turbojpeg.h"
45 
46 
47 /* Note: The following constant should not be changed. */
48 #define TIGHT_MIN_TO_COMPRESS 12
49 
50 /* The parameters below may be adjusted. */
51 #define MIN_SPLIT_RECT_SIZE 4096
52 #define MIN_SOLID_SUBRECT_SIZE 2048
53 #define MAX_SPLIT_TILE_SIZE 16
54 
55 /*
56  * There is so much access of the Tight encoding static data buffers
57  * that we resort to using thread local storage instead of having
58  * per-client data.
59  */
60 #if LIBVNCSERVER_HAVE_LIBPTHREAD && LIBVNCSERVER_HAVE_TLS && !defined(TLS) && defined(__linux__)
61 #define TLS __thread
62 #endif
63 #ifndef TLS
64 #define TLS
65 #endif
66 
67 /* This variable is set on every rfbSendRectEncodingTight() call. */
68 static TLS rfbBool usePixelFormat24 = FALSE;
69 
70 
71 /* Compression level stuff. The following array contains various
72  encoder parameters for each of 10 compression levels (0..9).
73  Last three parameters correspond to JPEG quality levels (0..9). */
74 
75 typedef struct TIGHT_CONF_s {
76  int maxRectSize, maxRectWidth;
78  int idxZlibLevel, monoZlibLevel, rawZlibLevel;
81 } TIGHT_CONF;
82 
83 static TIGHT_CONF tightConf[4] = {
84  { 65536, 2048, 6, 0, 0, 0, 4, 24 }, // 0 (used only without JPEG)
85  { 65536, 2048, 32, 1, 1, 1, 96, 24 }, // 1
86  { 65536, 2048, 32, 3, 3, 2, 96, 96 }, // 2 (used only with JPEG)
87  { 65536, 2048, 32, 7, 7, 5, 96, 256 } // 9
88 };
89 
90 #ifdef LIBVNCSERVER_HAVE_LIBPNG
91 typedef struct TIGHT_PNG_CONF_s {
92  int png_zlib_level, png_filters;
93 } TIGHT_PNG_CONF;
94 
95 static TIGHT_PNG_CONF tightPngConf[10] = {
96  { 0, PNG_NO_FILTERS },
97  { 1, PNG_NO_FILTERS },
98  { 2, PNG_NO_FILTERS },
99  { 3, PNG_NO_FILTERS },
100  { 4, PNG_NO_FILTERS },
101  { 5, PNG_ALL_FILTERS },
102  { 6, PNG_ALL_FILTERS },
103  { 7, PNG_ALL_FILTERS },
104  { 8, PNG_ALL_FILTERS },
105  { 9, PNG_ALL_FILTERS },
106 };
107 #endif
108 
109 static TLS int compressLevel = 1;
110 static TLS int qualityLevel = 95;
111 static TLS int subsampLevel = TJ_444;
112 
113 static const int subsampLevel2tjsubsamp[4] = {
114  TJ_444, TJ_420, TJ_422, TJ_GRAYSCALE
115 };
116 
117 
118 /* Stuff dealing with palettes. */
119 
120 typedef struct COLOR_LIST_s {
121  struct COLOR_LIST_s *next;
122  int idx;
123  uint32_t rgb;
124 } COLOR_LIST;
125 
126 typedef struct PALETTE_ENTRY_s {
129 } PALETTE_ENTRY;
130 
131 typedef struct PALETTE_s {
132  PALETTE_ENTRY entry[256];
133  COLOR_LIST *hash[256];
134  COLOR_LIST list[256];
135 } PALETTE;
136 
137 /* TODO: move into rfbScreen struct */
138 static TLS int paletteNumColors = 0;
139 static TLS int paletteMaxColors = 0;
140 static TLS uint32_t monoBackground = 0;
141 static TLS uint32_t monoForeground = 0;
142 static TLS PALETTE palette;
143 
144 /* Pointers to dynamically-allocated buffers. */
145 
146 static TLS int tightBeforeBufSize = 0;
147 static TLS char *tightBeforeBuf = NULL;
148 
149 static TLS int tightAfterBufSize = 0;
150 static TLS char *tightAfterBuf = NULL;
151 
152 static TLS tjhandle j = NULL;
153 
154 void rfbTightCleanup (rfbScreenInfoPtr screen)
155 {
156  if (tightBeforeBufSize) {
157  free (tightBeforeBuf);
158  tightBeforeBufSize = 0;
159  tightBeforeBuf = NULL;
160  }
161  if (tightAfterBufSize) {
162  free (tightAfterBuf);
163  tightAfterBufSize = 0;
164  tightAfterBuf = NULL;
165  }
166  if (j) tjDestroy(j);
167 }
168 
169 
170 /* Prototypes for static functions. */
171 
172 static rfbBool SendRectEncodingTight(rfbClientPtr cl, int x, int y,
173  int w, int h);
174 static void FindBestSolidArea (rfbClientPtr cl, int x, int y, int w, int h,
175  uint32_t colorValue, int *w_ptr, int *h_ptr);
176 static void ExtendSolidArea (rfbClientPtr cl, int x, int y, int w, int h,
177  uint32_t colorValue,
178  int *x_ptr, int *y_ptr, int *w_ptr, int *h_ptr);
179 static rfbBool CheckSolidTile (rfbClientPtr cl, int x, int y, int w, int h,
180  uint32_t *colorPtr, rfbBool needSameColor);
181 static rfbBool CheckSolidTile8 (rfbClientPtr cl, int x, int y, int w, int h,
182  uint32_t *colorPtr, rfbBool needSameColor);
183 static rfbBool CheckSolidTile16 (rfbClientPtr cl, int x, int y, int w, int h,
184  uint32_t *colorPtr, rfbBool needSameColor);
185 static rfbBool CheckSolidTile32 (rfbClientPtr cl, int x, int y, int w, int h,
186  uint32_t *colorPtr, rfbBool needSameColor);
187 
188 static rfbBool SendRectSimple (rfbClientPtr cl, int x, int y, int w, int h);
189 static rfbBool SendSubrect (rfbClientPtr cl, int x, int y, int w, int h);
190 static rfbBool SendTightHeader (rfbClientPtr cl, int x, int y, int w, int h);
191 
192 static rfbBool SendSolidRect (rfbClientPtr cl);
193 static rfbBool SendMonoRect (rfbClientPtr cl, int x, int y, int w, int h);
194 static rfbBool SendIndexedRect (rfbClientPtr cl, int x, int y, int w, int h);
195 static rfbBool SendFullColorRect (rfbClientPtr cl, int x, int y, int w, int h);
196 
197 static rfbBool CompressData (rfbClientPtr cl, int streamId, int dataLen,
198  int zlibLevel, int zlibStrategy);
199 static rfbBool SendCompressedData (rfbClientPtr cl, char *buf,
200  int compressedLen);
201 
202 static void FillPalette8 (int count);
203 static void FillPalette16 (int count);
204 static void FillPalette32 (int count);
205 static void FastFillPalette16 (rfbClientPtr cl, uint16_t *data, int w,
206  int pitch, int h);
207 static void FastFillPalette32 (rfbClientPtr cl, uint32_t *data, int w,
208  int pitch, int h);
209 
210 static void PaletteReset (void);
211 static int PaletteInsert (uint32_t rgb, int numPixels, int bpp);
212 
213 static void Pack24 (rfbClientPtr cl, char *buf, rfbPixelFormat *fmt,
214  int count);
215 
216 static void EncodeIndexedRect16 (uint8_t *buf, int count);
217 static void EncodeIndexedRect32 (uint8_t *buf, int count);
218 
219 static void EncodeMonoRect8 (uint8_t *buf, int w, int h);
220 static void EncodeMonoRect16 (uint8_t *buf, int w, int h);
221 static void EncodeMonoRect32 (uint8_t *buf, int w, int h);
222 
223 static rfbBool SendJpegRect (rfbClientPtr cl, int x, int y, int w, int h,
224  int quality);
225 static void PrepareRowForImg(rfbClientPtr cl, uint8_t *dst, int x, int y, int count);
226 static void PrepareRowForImg24(rfbClientPtr cl, uint8_t *dst, int x, int y, int count);
227 static void PrepareRowForImg16(rfbClientPtr cl, uint8_t *dst, int x, int y, int count);
228 static void PrepareRowForImg32(rfbClientPtr cl, uint8_t *dst, int x, int y, int count);
229 
230 #ifdef LIBVNCSERVER_HAVE_LIBPNG
231 static rfbBool SendPngRect(rfbClientPtr cl, int x, int y, int w, int h);
232 static rfbBool CanSendPngRect(rfbClientPtr cl, int w, int h);
233 #endif
234 
235 /*
236  * Tight encoding implementation.
237  */
238 
239 int
240 rfbNumCodedRectsTight(rfbClientPtr cl,
241  int x,
242  int y,
243  int w,
244  int h)
245 {
246  int maxRectSize, maxRectWidth;
247  int subrectMaxWidth, subrectMaxHeight;
248 
249  /* No matter how many rectangles we will send if LastRect markers
250  are used to terminate rectangle stream. */
251  if (cl->enableLastRectEncoding && w * h >= MIN_SPLIT_RECT_SIZE)
252  return 0;
253 
254  maxRectSize = tightConf[compressLevel].maxRectSize;
255  maxRectWidth = tightConf[compressLevel].maxRectWidth;
256 
257  if (w > maxRectWidth || w * h > maxRectSize) {
258  subrectMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
259  subrectMaxHeight = maxRectSize / subrectMaxWidth;
260  return (((w - 1) / maxRectWidth + 1) *
261  ((h - 1) / subrectMaxHeight + 1));
262  } else {
263  return 1;
264  }
265 }
266 
267 rfbBool
268 rfbSendRectEncodingTight(rfbClientPtr cl,
269  int x,
270  int y,
271  int w,
272  int h)
273 {
274  cl->tightEncoding = rfbEncodingTight;
275  return SendRectEncodingTight(cl, x, y, w, h);
276 }
277 
278 rfbBool
280  int x,
281  int y,
282  int w,
283  int h)
284 {
285  cl->tightEncoding = rfbEncodingTightPng;
286  return SendRectEncodingTight(cl, x, y, w, h);
287 }
288 
289 
290 rfbBool
291 SendRectEncodingTight(rfbClientPtr cl,
292  int x,
293  int y,
294  int w,
295  int h)
296 {
297  int nMaxRows;
298  uint32_t colorValue;
299  int dx, dy, dw, dh;
300  int x_best, y_best, w_best, h_best;
301  char *fbptr;
302 
303  rfbSendUpdateBuf(cl);
304 
305  compressLevel = cl->tightCompressLevel;
306  qualityLevel = cl->turboQualityLevel;
307  subsampLevel = cl->turboSubsampLevel;
308 
309  /* We only allow compression levels that have a demonstrable performance
310  benefit. CL 0 with JPEG reduces CPU usage for workloads that have low
311  numbers of unique colors, but the same thing can be accomplished by
312  using CL 0 without JPEG (AKA "Lossless Tight.") For those same
313  low-color workloads, CL 2 can provide typically 20-40% better
314  compression than CL 1 (with a commensurate increase in CPU usage.) For
315  high-color workloads, CL 1 should always be used, as higher compression
316  levels increase CPU usage for these workloads without providing any
317  significant reduction in bandwidth. */
318  if (qualityLevel != -1) {
319  if (compressLevel < 1) compressLevel = 1;
320  if (compressLevel > 2) compressLevel = 2;
321  }
322 
323  /* With JPEG disabled, CL 2 offers no significant bandwidth savings over
324  CL 1, so we don't include it. */
325  else if (compressLevel > 1) compressLevel = 1;
326 
327  /* CL 9 (which maps internally to CL 3) is included mainly for backward
328  compatibility with TightVNC Compression Levels 5-9. It should be used
329  only in extremely low-bandwidth cases in which it can be shown to have a
330  benefit. For low-color workloads, it provides typically only 10-20%
331  better compression than CL 2 with JPEG and CL 1 without JPEG, and it
332  uses, on average, twice as much CPU time. */
333  if (cl->tightCompressLevel == 9) compressLevel = 3;
334 
335  if ( cl->format.depth == 24 && cl->format.redMax == 0xFF &&
336  cl->format.greenMax == 0xFF && cl->format.blueMax == 0xFF ) {
337  usePixelFormat24 = TRUE;
338  } else {
339  usePixelFormat24 = FALSE;
340  }
341 
342  if (!cl->enableLastRectEncoding || w * h < MIN_SPLIT_RECT_SIZE)
343  return SendRectSimple(cl, x, y, w, h);
344 
345  /* Make sure we can write at least one pixel into tightBeforeBuf. */
346 
347  if (tightBeforeBufSize < 4) {
348  tightBeforeBufSize = 4;
349  if (tightBeforeBuf == NULL)
350  tightBeforeBuf = (char *)malloc(tightBeforeBufSize);
351  else
352  tightBeforeBuf = (char *)realloc(tightBeforeBuf,
353  tightBeforeBufSize);
354  }
355 
356  /* Calculate maximum number of rows in one non-solid rectangle. */
357 
358  {
359  int maxRectSize, maxRectWidth, nMaxWidth;
360 
361  maxRectSize = tightConf[compressLevel].maxRectSize;
362  maxRectWidth = tightConf[compressLevel].maxRectWidth;
363  nMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
364  nMaxRows = maxRectSize / nMaxWidth;
365  }
366 
367  /* Try to find large solid-color areas and send them separately. */
368 
369  for (dy = y; dy < y + h; dy += MAX_SPLIT_TILE_SIZE) {
370 
371  /* If a rectangle becomes too large, send its upper part now. */
372 
373  if (dy - y >= nMaxRows) {
374  if (!SendRectSimple(cl, x, y, w, nMaxRows))
375  return 0;
376  y += nMaxRows;
377  h -= nMaxRows;
378  }
379 
380  dh = (dy + MAX_SPLIT_TILE_SIZE <= y + h) ?
381  MAX_SPLIT_TILE_SIZE : (y + h - dy);
382 
383  for (dx = x; dx < x + w; dx += MAX_SPLIT_TILE_SIZE) {
384 
385  dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w) ?
386  MAX_SPLIT_TILE_SIZE : (x + w - dx);
387 
388  if (CheckSolidTile(cl, dx, dy, dw, dh, &colorValue, FALSE)) {
389 
390  if (subsampLevel == TJ_GRAYSCALE && qualityLevel != -1) {
391  uint32_t r = (colorValue >> 16) & 0xFF;
392  uint32_t g = (colorValue >> 8) & 0xFF;
393  uint32_t b = (colorValue) & 0xFF;
394  double y = (0.257 * (double)r) + (0.504 * (double)g)
395  + (0.098 * (double)b) + 16.;
396  colorValue = (int)y + (((int)y) << 8) + (((int)y) << 16);
397  }
398 
399  /* Get dimensions of solid-color area. */
400 
401  FindBestSolidArea(cl, dx, dy, w - (dx - x), h - (dy - y),
402  colorValue, &w_best, &h_best);
403 
404  /* Make sure a solid rectangle is large enough
405  (or the whole rectangle is of the same color). */
406 
407  if ( w_best * h_best != w * h &&
408  w_best * h_best < MIN_SOLID_SUBRECT_SIZE )
409  continue;
410 
411  /* Try to extend solid rectangle to maximum size. */
412 
413  x_best = dx; y_best = dy;
414  ExtendSolidArea(cl, x, y, w, h, colorValue,
415  &x_best, &y_best, &w_best, &h_best);
416 
417  /* Send rectangles at top and left to solid-color area. */
418 
419  if ( y_best != y &&
420  !SendRectSimple(cl, x, y, w, y_best-y) )
421  return FALSE;
422  if ( x_best != x &&
423  !SendRectEncodingTight(cl, x, y_best,
424  x_best-x, h_best) )
425  return FALSE;
426 
427  /* Send solid-color rectangle. */
428 
429  if (!SendTightHeader(cl, x_best, y_best, w_best, h_best))
430  return FALSE;
431 
432  fbptr = (cl->scaledScreen->frameBuffer +
433  (cl->scaledScreen->paddedWidthInBytes * y_best) +
434  (x_best * (cl->scaledScreen->bitsPerPixel / 8)));
435 
436  (*cl->translateFn)(cl->translateLookupTable, &cl->screen->serverFormat,
437  &cl->format, fbptr, tightBeforeBuf,
438  cl->scaledScreen->paddedWidthInBytes, 1, 1);
439 
440  if (!SendSolidRect(cl))
441  return FALSE;
442 
443  /* Send remaining rectangles (at right and bottom). */
444 
445  if ( x_best + w_best != x + w &&
446  !SendRectEncodingTight(cl, x_best + w_best, y_best,
447  w - (x_best-x) - w_best, h_best) )
448  return FALSE;
449  if ( y_best + h_best != y + h &&
450  !SendRectEncodingTight(cl, x, y_best + h_best,
451  w, h - (y_best-y) - h_best) )
452  return FALSE;
453 
454  /* Return after all recursive calls are done. */
455 
456  return TRUE;
457  }
458 
459  }
460 
461  }
462 
463  /* No suitable solid-color rectangles found. */
464 
465  return SendRectSimple(cl, x, y, w, h);
466 }
467 
468 
469 static void
470 FindBestSolidArea(rfbClientPtr cl,
471  int x,
472  int y,
473  int w,
474  int h,
475  uint32_t colorValue,
476  int *w_ptr,
477  int *h_ptr)
478 {
479  int dx, dy, dw, dh;
480  int w_prev;
481  int w_best = 0, h_best = 0;
482 
483  w_prev = w;
484 
485  for (dy = y; dy < y + h; dy += MAX_SPLIT_TILE_SIZE) {
486 
487  dh = (dy + MAX_SPLIT_TILE_SIZE <= y + h) ?
488  MAX_SPLIT_TILE_SIZE : (y + h - dy);
489  dw = (w_prev > MAX_SPLIT_TILE_SIZE) ?
490  MAX_SPLIT_TILE_SIZE : w_prev;
491 
492  if (!CheckSolidTile(cl, x, dy, dw, dh, &colorValue, TRUE))
493  break;
494 
495  for (dx = x + dw; dx < x + w_prev;) {
496  dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w_prev) ?
497  MAX_SPLIT_TILE_SIZE : (x + w_prev - dx);
498  if (!CheckSolidTile(cl, dx, dy, dw, dh, &colorValue, TRUE))
499  break;
500  dx += dw;
501  }
502 
503  w_prev = dx - x;
504  if (w_prev * (dy + dh - y) > w_best * h_best) {
505  w_best = w_prev;
506  h_best = dy + dh - y;
507  }
508  }
509 
510  *w_ptr = w_best;
511  *h_ptr = h_best;
512 }
513 
514 
515 static void
516 ExtendSolidArea(rfbClientPtr cl,
517  int x,
518  int y,
519  int w,
520  int h,
521  uint32_t colorValue,
522  int *x_ptr,
523  int *y_ptr,
524  int *w_ptr,
525  int *h_ptr)
526 {
527  int cx, cy;
528 
529  /* Try to extend the area upwards. */
530  for ( cy = *y_ptr - 1;
531  cy >= y && CheckSolidTile(cl, *x_ptr, cy, *w_ptr, 1, &colorValue, TRUE);
532  cy-- );
533  *h_ptr += *y_ptr - (cy + 1);
534  *y_ptr = cy + 1;
535 
536  /* ... downwards. */
537  for ( cy = *y_ptr + *h_ptr;
538  cy < y + h &&
539  CheckSolidTile(cl, *x_ptr, cy, *w_ptr, 1, &colorValue, TRUE);
540  cy++ );
541  *h_ptr += cy - (*y_ptr + *h_ptr);
542 
543  /* ... to the left. */
544  for ( cx = *x_ptr - 1;
545  cx >= x && CheckSolidTile(cl, cx, *y_ptr, 1, *h_ptr, &colorValue, TRUE);
546  cx-- );
547  *w_ptr += *x_ptr - (cx + 1);
548  *x_ptr = cx + 1;
549 
550  /* ... to the right. */
551  for ( cx = *x_ptr + *w_ptr;
552  cx < x + w &&
553  CheckSolidTile(cl, cx, *y_ptr, 1, *h_ptr, &colorValue, TRUE);
554  cx++ );
555  *w_ptr += cx - (*x_ptr + *w_ptr);
556 }
557 
558 
559 /*
560  * Check if a rectangle is all of the same color. If needSameColor is
561  * set to non-zero, then also check that its color equals to the
562  * *colorPtr value. The result is 1 if the test is successfull, and in
563  * that case new color will be stored in *colorPtr.
564  */
565 
566 static rfbBool CheckSolidTile(rfbClientPtr cl, int x, int y, int w, int h, uint32_t* colorPtr, rfbBool needSameColor)
567 {
568  switch(cl->screen->serverFormat.bitsPerPixel) {
569  case 32:
570  return CheckSolidTile32(cl, x, y, w, h, colorPtr, needSameColor);
571  case 16:
572  return CheckSolidTile16(cl, x, y, w, h, colorPtr, needSameColor);
573  default:
574  return CheckSolidTile8(cl, x, y, w, h, colorPtr, needSameColor);
575  }
576 }
577 
578 
579 #define DEFINE_CHECK_SOLID_FUNCTION(bpp) \
580  \
581 static rfbBool \
582 CheckSolidTile##bpp(rfbClientPtr cl, int x, int y, int w, int h, \
583  uint32_t* colorPtr, rfbBool needSameColor) \
584 { \
585  uint##bpp##_t *fbptr; \
586  uint##bpp##_t colorValue; \
587  int dx, dy; \
588  \
589  fbptr = (uint##bpp##_t *)&cl->scaledScreen->frameBuffer \
590  [y * cl->scaledScreen->paddedWidthInBytes + x * (bpp/8)]; \
591  \
592  colorValue = *fbptr; \
593  if (needSameColor && (uint32_t)colorValue != *colorPtr) \
594  return FALSE; \
595  \
596  for (dy = 0; dy < h; dy++) { \
597  for (dx = 0; dx < w; dx++) { \
598  if (colorValue != fbptr[dx]) \
599  return FALSE; \
600  } \
601  fbptr = (uint##bpp##_t *)((uint8_t *)fbptr \
602  + cl->scaledScreen->paddedWidthInBytes); \
603  } \
604  \
605  *colorPtr = (uint32_t)colorValue; \
606  return TRUE; \
607 }
608 
612 
613 static rfbBool
614 SendRectSimple(rfbClientPtr cl, int x, int y, int w, int h)
615 {
616  int maxBeforeSize, maxAfterSize;
617  int maxRectSize, maxRectWidth;
618  int subrectMaxWidth, subrectMaxHeight;
619  int dx, dy;
620  int rw, rh;
621 
622  maxRectSize = tightConf[compressLevel].maxRectSize;
623  maxRectWidth = tightConf[compressLevel].maxRectWidth;
624 
625  maxBeforeSize = maxRectSize * (cl->format.bitsPerPixel / 8);
626  maxAfterSize = maxBeforeSize + (maxBeforeSize + 99) / 100 + 12;
627 
628  if (tightBeforeBufSize < maxBeforeSize) {
629  tightBeforeBufSize = maxBeforeSize;
630  if (tightBeforeBuf == NULL)
631  tightBeforeBuf = (char *)malloc(tightBeforeBufSize);
632  else
633  tightBeforeBuf = (char *)realloc(tightBeforeBuf,
634  tightBeforeBufSize);
635  }
636 
637  if (tightAfterBufSize < maxAfterSize) {
638  tightAfterBufSize = maxAfterSize;
639  if (tightAfterBuf == NULL)
640  tightAfterBuf = (char *)malloc(tightAfterBufSize);
641  else
642  tightAfterBuf = (char *)realloc(tightAfterBuf,
643  tightAfterBufSize);
644  }
645 
646  if (w > maxRectWidth || w * h > maxRectSize) {
647  subrectMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
648  subrectMaxHeight = maxRectSize / subrectMaxWidth;
649 
650  for (dy = 0; dy < h; dy += subrectMaxHeight) {
651  for (dx = 0; dx < w; dx += maxRectWidth) {
652  rw = (dx + maxRectWidth < w) ? maxRectWidth : w - dx;
653  rh = (dy + subrectMaxHeight < h) ? subrectMaxHeight : h - dy;
654  if (!SendSubrect(cl, x + dx, y + dy, rw, rh))
655  return FALSE;
656  }
657  }
658  } else {
659  if (!SendSubrect(cl, x, y, w, h))
660  return FALSE;
661  }
662 
663  return TRUE;
664 }
665 
666 static rfbBool
667 SendSubrect(rfbClientPtr cl,
668  int x,
669  int y,
670  int w,
671  int h)
672 {
673  char *fbptr;
674  rfbBool success = FALSE;
675 
676  /* Send pending data if there is more than 128 bytes. */
677  if (cl->ublen > 128) {
678  if (!rfbSendUpdateBuf(cl))
679  return FALSE;
680  }
681 
682  if (!SendTightHeader(cl, x, y, w, h))
683  return FALSE;
684 
685  fbptr = (cl->scaledScreen->frameBuffer
686  + (cl->scaledScreen->paddedWidthInBytes * y)
687  + (x * (cl->scaledScreen->bitsPerPixel / 8)));
688 
689  if (subsampLevel == TJ_GRAYSCALE && qualityLevel != -1)
690  return SendJpegRect(cl, x, y, w, h, qualityLevel);
691 
692  paletteMaxColors = w * h / tightConf[compressLevel].idxMaxColorsDivisor;
693  if(qualityLevel != -1)
694  paletteMaxColors = tightConf[compressLevel].palMaxColorsWithJPEG;
695  if ( paletteMaxColors < 2 &&
696  w * h >= tightConf[compressLevel].monoMinRectSize ) {
697  paletteMaxColors = 2;
698  }
699 
700  if (cl->format.bitsPerPixel == cl->screen->serverFormat.bitsPerPixel &&
701  cl->format.redMax == cl->screen->serverFormat.redMax &&
702  cl->format.greenMax == cl->screen->serverFormat.greenMax &&
703  cl->format.blueMax == cl->screen->serverFormat.blueMax &&
704  cl->format.bitsPerPixel >= 16) {
705 
706  /* This is so we can avoid translating the pixels when compressing
707  with JPEG, since it is unnecessary */
708  switch (cl->format.bitsPerPixel) {
709  case 16:
710  FastFillPalette16(cl, (uint16_t *)fbptr, w,
711  cl->scaledScreen->paddedWidthInBytes / 2, h);
712  break;
713  default:
714  FastFillPalette32(cl, (uint32_t *)fbptr, w,
715  cl->scaledScreen->paddedWidthInBytes / 4, h);
716  }
717 
718  if(paletteNumColors != 0 || qualityLevel == -1) {
719  (*cl->translateFn)(cl->translateLookupTable,
720  &cl->screen->serverFormat, &cl->format, fbptr,
721  tightBeforeBuf,
722  cl->scaledScreen->paddedWidthInBytes, w, h);
723  }
724  }
725  else {
726  (*cl->translateFn)(cl->translateLookupTable, &cl->screen->serverFormat,
727  &cl->format, fbptr, tightBeforeBuf,
728  cl->scaledScreen->paddedWidthInBytes, w, h);
729 
730  switch (cl->format.bitsPerPixel) {
731  case 8:
732  FillPalette8(w * h);
733  break;
734  case 16:
735  FillPalette16(w * h);
736  break;
737  default:
738  FillPalette32(w * h);
739  }
740  }
741 
742  switch (paletteNumColors) {
743  case 0:
744  /* Truecolor image */
745  if (qualityLevel != -1) {
746  success = SendJpegRect(cl, x, y, w, h, qualityLevel);
747  } else {
748  success = SendFullColorRect(cl, x, y, w, h);
749  }
750  break;
751  case 1:
752  /* Solid rectangle */
753  success = SendSolidRect(cl);
754  break;
755  case 2:
756  /* Two-color rectangle */
757  success = SendMonoRect(cl, x, y, w, h);
758  break;
759  default:
760  /* Up to 256 different colors */
761  success = SendIndexedRect(cl, x, y, w, h);
762  }
763  return success;
764 }
765 
766 static rfbBool
767 SendTightHeader(rfbClientPtr cl,
768  int x,
769  int y,
770  int w,
771  int h)
772 {
774 
776  if (!rfbSendUpdateBuf(cl))
777  return FALSE;
778  }
779 
780  rect.r.x = Swap16IfLE(x);
781  rect.r.y = Swap16IfLE(y);
782  rect.r.w = Swap16IfLE(w);
783  rect.r.h = Swap16IfLE(h);
784  rect.encoding = Swap32IfLE(cl->tightEncoding);
785 
786  memcpy(&cl->updateBuf[cl->ublen], (char *)&rect,
789 
790  rfbStatRecordEncodingSent(cl, cl->tightEncoding,
793  + w * (cl->format.bitsPerPixel / 8) * h);
794 
795  return TRUE;
796 }
797 
798 /*
799  * Subencoding implementations.
800  */
801 
802 static rfbBool
803 SendSolidRect(rfbClientPtr cl)
804 {
805  int len;
806 
807  if (usePixelFormat24) {
808  Pack24(cl, tightBeforeBuf, &cl->format, 1);
809  len = 3;
810  } else
811  len = cl->format.bitsPerPixel / 8;
812 
813  if (cl->ublen + 1 + len > UPDATE_BUF_SIZE) {
814  if (!rfbSendUpdateBuf(cl))
815  return FALSE;
816  }
817 
818  cl->updateBuf[cl->ublen++] = (char)(rfbTightFill << 4);
819  memcpy (&cl->updateBuf[cl->ublen], tightBeforeBuf, len);
820  cl->ublen += len;
821 
822  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding, len + 1);
823 
824  return TRUE;
825 }
826 
827 static rfbBool
828 SendMonoRect(rfbClientPtr cl,
829  int x,
830  int y,
831  int w,
832  int h)
833 {
834  int streamId = 1;
835  int paletteLen, dataLen;
836 
837 #ifdef LIBVNCSERVER_HAVE_LIBPNG
838  if (CanSendPngRect(cl, w, h)) {
839  /* TODO: setup palette maybe */
840  return SendPngRect(cl, x, y, w, h);
841  /* TODO: destroy palette maybe */
842  }
843 #endif
844 
845  if ( cl->ublen + TIGHT_MIN_TO_COMPRESS + 6 +
846  2 * cl->format.bitsPerPixel / 8 > UPDATE_BUF_SIZE ) {
847  if (!rfbSendUpdateBuf(cl))
848  return FALSE;
849  }
850 
851  /* Prepare tight encoding header. */
852  dataLen = (w + 7) / 8;
853  dataLen *= h;
854 
855  if (tightConf[compressLevel].monoZlibLevel == 0 &&
856  cl->tightEncoding != rfbEncodingTightPng)
857  cl->updateBuf[cl->ublen++] =
858  (char)((rfbTightNoZlib | rfbTightExplicitFilter) << 4);
859  else
860  cl->updateBuf[cl->ublen++] = (streamId | rfbTightExplicitFilter) << 4;
861  cl->updateBuf[cl->ublen++] = rfbTightFilterPalette;
862  cl->updateBuf[cl->ublen++] = 1;
863 
864  /* Prepare palette, convert image. */
865  switch (cl->format.bitsPerPixel) {
866 
867  case 32:
868  EncodeMonoRect32((uint8_t *)tightBeforeBuf, w, h);
869 
870  ((uint32_t *)tightAfterBuf)[0] = monoBackground;
871  ((uint32_t *)tightAfterBuf)[1] = monoForeground;
872  if (usePixelFormat24) {
873  Pack24(cl, tightAfterBuf, &cl->format, 2);
874  paletteLen = 6;
875  } else
876  paletteLen = 8;
877 
878  memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, paletteLen);
879  cl->ublen += paletteLen;
880  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding, 3 + paletteLen);
881  break;
882 
883  case 16:
884  EncodeMonoRect16((uint8_t *)tightBeforeBuf, w, h);
885 
886  ((uint16_t *)tightAfterBuf)[0] = (uint16_t)monoBackground;
887  ((uint16_t *)tightAfterBuf)[1] = (uint16_t)monoForeground;
888 
889  memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, 4);
890  cl->ublen += 4;
891  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding, 7);
892  break;
893 
894  default:
895  EncodeMonoRect8((uint8_t *)tightBeforeBuf, w, h);
896 
897  cl->updateBuf[cl->ublen++] = (char)monoBackground;
898  cl->updateBuf[cl->ublen++] = (char)monoForeground;
899  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding, 5);
900  }
901 
902  return CompressData(cl, streamId, dataLen,
903  tightConf[compressLevel].monoZlibLevel,
904  Z_DEFAULT_STRATEGY);
905 }
906 
907 static rfbBool
908 SendIndexedRect(rfbClientPtr cl,
909  int x,
910  int y,
911  int w,
912  int h)
913 {
914  int streamId = 2;
915  int i, entryLen;
916 
917 #ifdef LIBVNCSERVER_HAVE_LIBPNG
918  if (CanSendPngRect(cl, w, h)) {
919  return SendPngRect(cl, x, y, w, h);
920  }
921 #endif
922 
923  if ( cl->ublen + TIGHT_MIN_TO_COMPRESS + 6 +
924  paletteNumColors * cl->format.bitsPerPixel / 8 >
925  UPDATE_BUF_SIZE ) {
926  if (!rfbSendUpdateBuf(cl))
927  return FALSE;
928  }
929 
930  /* Prepare tight encoding header. */
931  if (tightConf[compressLevel].idxZlibLevel == 0 &&
932  cl->tightEncoding != rfbEncodingTightPng)
933  cl->updateBuf[cl->ublen++] =
934  (char)((rfbTightNoZlib | rfbTightExplicitFilter) << 4);
935  else
936  cl->updateBuf[cl->ublen++] = (streamId | rfbTightExplicitFilter) << 4;
937  cl->updateBuf[cl->ublen++] = rfbTightFilterPalette;
938  cl->updateBuf[cl->ublen++] = (char)(paletteNumColors - 1);
939 
940  /* Prepare palette, convert image. */
941  switch (cl->format.bitsPerPixel) {
942 
943  case 32:
944  EncodeIndexedRect32((uint8_t *)tightBeforeBuf, w * h);
945 
946  for (i = 0; i < paletteNumColors; i++) {
947  ((uint32_t *)tightAfterBuf)[i] =
948  palette.entry[i].listNode->rgb;
949  }
950  if (usePixelFormat24) {
951  Pack24(cl, tightAfterBuf, &cl->format, paletteNumColors);
952  entryLen = 3;
953  } else
954  entryLen = 4;
955 
956  memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf,
957  paletteNumColors * entryLen);
958  cl->ublen += paletteNumColors * entryLen;
959  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding,
960  3 + paletteNumColors * entryLen);
961  break;
962 
963  case 16:
964  EncodeIndexedRect16((uint8_t *)tightBeforeBuf, w * h);
965 
966  for (i = 0; i < paletteNumColors; i++) {
967  ((uint16_t *)tightAfterBuf)[i] =
968  (uint16_t)palette.entry[i].listNode->rgb;
969  }
970 
971  memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, paletteNumColors * 2);
972  cl->ublen += paletteNumColors * 2;
973  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding,
974  3 + paletteNumColors * 2);
975  break;
976 
977  default:
978  return FALSE; /* Should never happen. */
979  }
980 
981  return CompressData(cl, streamId, w * h,
982  tightConf[compressLevel].idxZlibLevel,
983  Z_DEFAULT_STRATEGY);
984 }
985 
986 static rfbBool
987 SendFullColorRect(rfbClientPtr cl,
988  int x,
989  int y,
990  int w,
991  int h)
992 {
993  int streamId = 0;
994  int len;
995 
996 #ifdef LIBVNCSERVER_HAVE_LIBPNG
997  if (CanSendPngRect(cl, w, h)) {
998  return SendPngRect(cl, x, y, w, h);
999  }
1000 #endif
1001 
1002  if (cl->ublen + TIGHT_MIN_TO_COMPRESS + 1 > UPDATE_BUF_SIZE) {
1003  if (!rfbSendUpdateBuf(cl))
1004  return FALSE;
1005  }
1006 
1007  if (tightConf[compressLevel].rawZlibLevel == 0 &&
1008  cl->tightEncoding != rfbEncodingTightPng)
1009  cl->updateBuf[cl->ublen++] = (char)(rfbTightNoZlib << 4);
1010  else
1011  cl->updateBuf[cl->ublen++] = 0x00; /* stream id = 0, no flushing, no filter */
1012  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding, 1);
1013 
1014  if (usePixelFormat24) {
1015  Pack24(cl, tightBeforeBuf, &cl->format, w * h);
1016  len = 3;
1017  } else
1018  len = cl->format.bitsPerPixel / 8;
1019 
1020  return CompressData(cl, streamId, w * h * len,
1021  tightConf[compressLevel].rawZlibLevel,
1022  Z_DEFAULT_STRATEGY);
1023 }
1024 
1025 static rfbBool
1026 CompressData(rfbClientPtr cl,
1027  int streamId,
1028  int dataLen,
1029  int zlibLevel,
1030  int zlibStrategy)
1031 {
1032  z_streamp pz;
1033  int err;
1034 
1035  if (dataLen < TIGHT_MIN_TO_COMPRESS) {
1036  memcpy(&cl->updateBuf[cl->ublen], tightBeforeBuf, dataLen);
1037  cl->ublen += dataLen;
1038  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding, dataLen);
1039  return TRUE;
1040  }
1041 
1042  if (zlibLevel == 0)
1043  return SendCompressedData (cl, tightBeforeBuf, dataLen);
1044 
1045  pz = &cl->zsStruct[streamId];
1046 
1047  /* Initialize compression stream if needed. */
1048  if (!cl->zsActive[streamId]) {
1049  pz->zalloc = Z_NULL;
1050  pz->zfree = Z_NULL;
1051  pz->opaque = Z_NULL;
1052 
1053  err = deflateInit2 (pz, zlibLevel, Z_DEFLATED, MAX_WBITS,
1054  MAX_MEM_LEVEL, zlibStrategy);
1055  if (err != Z_OK)
1056  return FALSE;
1057 
1058  cl->zsActive[streamId] = TRUE;
1059  cl->zsLevel[streamId] = zlibLevel;
1060  }
1061 
1062  /* Prepare buffer pointers. */
1063  pz->next_in = (Bytef *)tightBeforeBuf;
1064  pz->avail_in = dataLen;
1065  pz->next_out = (Bytef *)tightAfterBuf;
1066  pz->avail_out = tightAfterBufSize;
1067 
1068  /* Change compression parameters if needed. */
1069  if (zlibLevel != cl->zsLevel[streamId]) {
1070  if (deflateParams (pz, zlibLevel, zlibStrategy) != Z_OK) {
1071  return FALSE;
1072  }
1073  cl->zsLevel[streamId] = zlibLevel;
1074  }
1075 
1076  /* Actual compression. */
1077  if (deflate(pz, Z_SYNC_FLUSH) != Z_OK ||
1078  pz->avail_in != 0 || pz->avail_out == 0) {
1079  return FALSE;
1080  }
1081 
1082  return SendCompressedData(cl, tightAfterBuf,
1083  tightAfterBufSize - pz->avail_out);
1084 }
1085 
1086 static rfbBool SendCompressedData(rfbClientPtr cl, char *buf,
1087  int compressedLen)
1088 {
1089  int i, portionLen;
1090 
1091  cl->updateBuf[cl->ublen++] = compressedLen & 0x7F;
1092  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding, 1);
1093  if (compressedLen > 0x7F) {
1094  cl->updateBuf[cl->ublen-1] |= 0x80;
1095  cl->updateBuf[cl->ublen++] = compressedLen >> 7 & 0x7F;
1096  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding, 1);
1097  if (compressedLen > 0x3FFF) {
1098  cl->updateBuf[cl->ublen-1] |= 0x80;
1099  cl->updateBuf[cl->ublen++] = compressedLen >> 14 & 0xFF;
1100  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding, 1);
1101  }
1102  }
1103 
1104  portionLen = UPDATE_BUF_SIZE;
1105  for (i = 0; i < compressedLen; i += portionLen) {
1106  if (i + portionLen > compressedLen) {
1107  portionLen = compressedLen - i;
1108  }
1109  if (cl->ublen + portionLen > UPDATE_BUF_SIZE) {
1110  if (!rfbSendUpdateBuf(cl))
1111  return FALSE;
1112  }
1113  memcpy(&cl->updateBuf[cl->ublen], &buf[i], portionLen);
1114  cl->ublen += portionLen;
1115  }
1116  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding, compressedLen);
1117 
1118  return TRUE;
1119 }
1120 
1121 
1122 /*
1123  * Code to determine how many different colors used in rectangle.
1124  */
1125 
1126 static void
1127 FillPalette8(int count)
1128 {
1129  uint8_t *data = (uint8_t *)tightBeforeBuf;
1130  uint8_t c0, c1;
1131  int i, n0, n1;
1132 
1133  paletteNumColors = 0;
1134 
1135  c0 = data[0];
1136  for (i = 1; i < count && data[i] == c0; i++);
1137  if (i == count) {
1138  paletteNumColors = 1;
1139  return; /* Solid rectangle */
1140  }
1141 
1142  if (paletteMaxColors < 2)
1143  return;
1144 
1145  n0 = i;
1146  c1 = data[i];
1147  n1 = 0;
1148  for (i++; i < count; i++) {
1149  if (data[i] == c0) {
1150  n0++;
1151  } else if (data[i] == c1) {
1152  n1++;
1153  } else
1154  break;
1155  }
1156  if (i == count) {
1157  if (n0 > n1) {
1158  monoBackground = (uint32_t)c0;
1159  monoForeground = (uint32_t)c1;
1160  } else {
1161  monoBackground = (uint32_t)c1;
1162  monoForeground = (uint32_t)c0;
1163  }
1164  paletteNumColors = 2; /* Two colors */
1165  }
1166 }
1167 
1168 
1169 #define DEFINE_FILL_PALETTE_FUNCTION(bpp) \
1170  \
1171 static void \
1172 FillPalette##bpp(int count) { \
1173  uint##bpp##_t *data = (uint##bpp##_t *)tightBeforeBuf; \
1174  uint##bpp##_t c0, c1, ci; \
1175  int i, n0, n1, ni; \
1176  \
1177  c0 = data[0]; \
1178  for (i = 1; i < count && data[i] == c0; i++); \
1179  if (i >= count) { \
1180  paletteNumColors = 1; /* Solid rectangle */ \
1181  return; \
1182  } \
1183  \
1184  if (paletteMaxColors < 2) { \
1185  paletteNumColors = 0; /* Full-color encoding preferred */ \
1186  return; \
1187  } \
1188  \
1189  n0 = i; \
1190  c1 = data[i]; \
1191  n1 = 0; \
1192  for (i++; i < count; i++) { \
1193  ci = data[i]; \
1194  if (ci == c0) { \
1195  n0++; \
1196  } else if (ci == c1) { \
1197  n1++; \
1198  } else \
1199  break; \
1200  } \
1201  if (i >= count) { \
1202  if (n0 > n1) { \
1203  monoBackground = (uint32_t)c0; \
1204  monoForeground = (uint32_t)c1; \
1205  } else { \
1206  monoBackground = (uint32_t)c1; \
1207  monoForeground = (uint32_t)c0; \
1208  } \
1209  paletteNumColors = 2; /* Two colors */ \
1210  return; \
1211  } \
1212  \
1213  PaletteReset(); \
1214  PaletteInsert (c0, (uint32_t)n0, bpp); \
1215  PaletteInsert (c1, (uint32_t)n1, bpp); \
1216  \
1217  ni = 1; \
1218  for (i++; i < count; i++) { \
1219  if (data[i] == ci) { \
1220  ni++; \
1221  } else { \
1222  if (!PaletteInsert (ci, (uint32_t)ni, bpp)) \
1223  return; \
1224  ci = data[i]; \
1225  ni = 1; \
1226  } \
1227  } \
1228  PaletteInsert (ci, (uint32_t)ni, bpp); \
1229 }
1230 
1233 
1234 #define DEFINE_FAST_FILL_PALETTE_FUNCTION(bpp) \
1235  \
1236 static void \
1237 FastFillPalette##bpp(rfbClientPtr cl, uint##bpp##_t *data, int w, \
1238  int pitch, int h) \
1239 { \
1240  uint##bpp##_t c0, c1, ci, mask, c0t, c1t, cit; \
1241  int i, j, i2 = 0, j2, n0, n1, ni; \
1242  \
1243  if (cl->translateFn != rfbTranslateNone) { \
1244  mask = cl->screen->serverFormat.redMax \
1245  << cl->screen->serverFormat.redShift; \
1246  mask |= cl->screen->serverFormat.greenMax \
1247  << cl->screen->serverFormat.greenShift; \
1248  mask |= cl->screen->serverFormat.blueMax \
1249  << cl->screen->serverFormat.blueShift; \
1250  } else mask = ~0; \
1251  \
1252  c0 = data[0] & mask; \
1253  for (j = 0; j < h; j++) { \
1254  for (i = 0; i < w; i++) { \
1255  if ((data[j * pitch + i] & mask) != c0) \
1256  goto done; \
1257  } \
1258  } \
1259  done: \
1260  if (j >= h) { \
1261  paletteNumColors = 1; /* Solid rectangle */ \
1262  return; \
1263  } \
1264  if (paletteMaxColors < 2) { \
1265  paletteNumColors = 0; /* Full-color encoding preferred */ \
1266  return; \
1267  } \
1268  \
1269  n0 = j * w + i; \
1270  c1 = data[j * pitch + i] & mask; \
1271  n1 = 0; \
1272  i++; if (i >= w) {i = 0; j++;} \
1273  for (j2 = j; j2 < h; j2++) { \
1274  for (i2 = i; i2 < w; i2++) { \
1275  ci = data[j2 * pitch + i2] & mask; \
1276  if (ci == c0) { \
1277  n0++; \
1278  } else if (ci == c1) { \
1279  n1++; \
1280  } else \
1281  goto done2; \
1282  } \
1283  i = 0; \
1284  } \
1285  done2: \
1286  (*cl->translateFn)(cl->translateLookupTable, \
1287  &cl->screen->serverFormat, &cl->format, \
1288  (char *)&c0, (char *)&c0t, bpp/8, 1, 1); \
1289  (*cl->translateFn)(cl->translateLookupTable, \
1290  &cl->screen->serverFormat, &cl->format, \
1291  (char *)&c1, (char *)&c1t, bpp/8, 1, 1); \
1292  if (j2 >= h) { \
1293  if (n0 > n1) { \
1294  monoBackground = (uint32_t)c0t; \
1295  monoForeground = (uint32_t)c1t; \
1296  } else { \
1297  monoBackground = (uint32_t)c1t; \
1298  monoForeground = (uint32_t)c0t; \
1299  } \
1300  paletteNumColors = 2; /* Two colors */ \
1301  return; \
1302  } \
1303  \
1304  PaletteReset(); \
1305  PaletteInsert (c0t, (uint32_t)n0, bpp); \
1306  PaletteInsert (c1t, (uint32_t)n1, bpp); \
1307  \
1308  ni = 1; \
1309  i2++; if (i2 >= w) {i2 = 0; j2++;} \
1310  for (j = j2; j < h; j++) { \
1311  for (i = i2; i < w; i++) { \
1312  if ((data[j * pitch + i] & mask) == ci) { \
1313  ni++; \
1314  } else { \
1315  (*cl->translateFn)(cl->translateLookupTable, \
1316  &cl->screen->serverFormat, \
1317  &cl->format, (char *)&ci, \
1318  (char *)&cit, bpp/8, 1, 1); \
1319  if (!PaletteInsert (cit, (uint32_t)ni, bpp)) \
1320  return; \
1321  ci = data[j * pitch + i] & mask; \
1322  ni = 1; \
1323  } \
1324  } \
1325  i2 = 0; \
1326  } \
1327  \
1328  (*cl->translateFn)(cl->translateLookupTable, \
1329  &cl->screen->serverFormat, &cl->format, \
1330  (char *)&ci, (char *)&cit, bpp/8, 1, 1); \
1331  PaletteInsert (cit, (uint32_t)ni, bpp); \
1332 }
1333 
1336 
1337 
1338 /*
1339  * Functions to operate with palette structures.
1340  */
1341 
1342 #define HASH_FUNC16(rgb) ((int)((((rgb) >> 8) + (rgb)) & 0xFF))
1343 #define HASH_FUNC32(rgb) ((int)((((rgb) >> 16) + ((rgb) >> 8)) & 0xFF))
1344 
1345 
1346 static void
1347 PaletteReset(void)
1348 {
1349  paletteNumColors = 0;
1350  memset(palette.hash, 0, 256 * sizeof(COLOR_LIST *));
1351 }
1352 
1353 
1354 static int
1355 PaletteInsert(uint32_t rgb,
1356  int numPixels,
1357  int bpp)
1358 {
1359  COLOR_LIST *pnode;
1360  COLOR_LIST *prev_pnode = NULL;
1361  int hash_key, idx, new_idx, count;
1362 
1363  hash_key = (bpp == 16) ? HASH_FUNC16(rgb) : HASH_FUNC32(rgb);
1364 
1365  pnode = palette.hash[hash_key];
1366 
1367  while (pnode != NULL) {
1368  if (pnode->rgb == rgb) {
1369  /* Such palette entry already exists. */
1370  new_idx = idx = pnode->idx;
1371  count = palette.entry[idx].numPixels + numPixels;
1372  if (new_idx && palette.entry[new_idx-1].numPixels < count) {
1373  do {
1374  palette.entry[new_idx] = palette.entry[new_idx-1];
1375  palette.entry[new_idx].listNode->idx = new_idx;
1376  new_idx--;
1377  }
1378  while (new_idx && palette.entry[new_idx-1].numPixels < count);
1379  palette.entry[new_idx].listNode = pnode;
1380  pnode->idx = new_idx;
1381  }
1382  palette.entry[new_idx].numPixels = count;
1383  return paletteNumColors;
1384  }
1385  prev_pnode = pnode;
1386  pnode = pnode->next;
1387  }
1388 
1389  /* Check if palette is full. */
1390  if (paletteNumColors == 256 || paletteNumColors == paletteMaxColors) {
1391  paletteNumColors = 0;
1392  return 0;
1393  }
1394 
1395  /* Move palette entries with lesser pixel counts. */
1396  for ( idx = paletteNumColors;
1397  idx > 0 && palette.entry[idx-1].numPixels < numPixels;
1398  idx-- ) {
1399  palette.entry[idx] = palette.entry[idx-1];
1400  palette.entry[idx].listNode->idx = idx;
1401  }
1402 
1403  /* Add new palette entry into the freed slot. */
1404  pnode = &palette.list[paletteNumColors];
1405  if (prev_pnode != NULL) {
1406  prev_pnode->next = pnode;
1407  } else {
1408  palette.hash[hash_key] = pnode;
1409  }
1410  pnode->next = NULL;
1411  pnode->idx = idx;
1412  pnode->rgb = rgb;
1413  palette.entry[idx].listNode = pnode;
1414  palette.entry[idx].numPixels = numPixels;
1415 
1416  return (++paletteNumColors);
1417 }
1418 
1419 
1420 /*
1421  * Converting 32-bit color samples into 24-bit colors.
1422  * Should be called only when redMax, greenMax and blueMax are 255.
1423  * Color components assumed to be byte-aligned.
1424  */
1425 
1426 static void Pack24(rfbClientPtr cl,
1427  char *buf,
1428  rfbPixelFormat *fmt,
1429  int count)
1430 {
1431  uint32_t *buf32;
1432  uint32_t pix;
1433  int r_shift, g_shift, b_shift;
1434 
1435  buf32 = (uint32_t *)buf;
1436 
1437  if (!cl->screen->serverFormat.bigEndian == !fmt->bigEndian) {
1438  r_shift = fmt->redShift;
1439  g_shift = fmt->greenShift;
1440  b_shift = fmt->blueShift;
1441  } else {
1442  r_shift = 24 - fmt->redShift;
1443  g_shift = 24 - fmt->greenShift;
1444  b_shift = 24 - fmt->blueShift;
1445  }
1446 
1447  while (count--) {
1448  pix = *buf32++;
1449  *buf++ = (char)(pix >> r_shift);
1450  *buf++ = (char)(pix >> g_shift);
1451  *buf++ = (char)(pix >> b_shift);
1452  }
1453 }
1454 
1455 
1456 /*
1457  * Converting truecolor samples into palette indices.
1458  */
1459 
1460 #define DEFINE_IDX_ENCODE_FUNCTION(bpp) \
1461  \
1462 static void \
1463 EncodeIndexedRect##bpp(uint8_t *buf, int count) { \
1464  COLOR_LIST *pnode; \
1465  uint##bpp##_t *src; \
1466  uint##bpp##_t rgb; \
1467  int rep = 0; \
1468  \
1469  src = (uint##bpp##_t *) buf; \
1470  \
1471  while (count--) { \
1472  rgb = *src++; \
1473  while (count && *src == rgb) { \
1474  rep++, src++, count--; \
1475  } \
1476  pnode = palette.hash[HASH_FUNC##bpp(rgb)]; \
1477  while (pnode != NULL) { \
1478  if ((uint##bpp##_t)pnode->rgb == rgb) { \
1479  *buf++ = (uint8_t)pnode->idx; \
1480  while (rep) { \
1481  *buf++ = (uint8_t)pnode->idx; \
1482  rep--; \
1483  } \
1484  break; \
1485  } \
1486  pnode = pnode->next; \
1487  } \
1488  } \
1489 }
1490 
1493 
1494 
1495 #define DEFINE_MONO_ENCODE_FUNCTION(bpp) \
1496  \
1497 static void \
1498 EncodeMonoRect##bpp(uint8_t *buf, int w, int h) { \
1499  uint##bpp##_t *ptr; \
1500  uint##bpp##_t bg; \
1501  unsigned int value, mask; \
1502  int aligned_width; \
1503  int x, y, bg_bits; \
1504  \
1505  ptr = (uint##bpp##_t *) buf; \
1506  bg = (uint##bpp##_t) monoBackground; \
1507  aligned_width = w - w % 8; \
1508  \
1509  for (y = 0; y < h; y++) { \
1510  for (x = 0; x < aligned_width; x += 8) { \
1511  for (bg_bits = 0; bg_bits < 8; bg_bits++) { \
1512  if (*ptr++ != bg) \
1513  break; \
1514  } \
1515  if (bg_bits == 8) { \
1516  *buf++ = 0; \
1517  continue; \
1518  } \
1519  mask = 0x80 >> bg_bits; \
1520  value = mask; \
1521  for (bg_bits++; bg_bits < 8; bg_bits++) { \
1522  mask >>= 1; \
1523  if (*ptr++ != bg) { \
1524  value |= mask; \
1525  } \
1526  } \
1527  *buf++ = (uint8_t)value; \
1528  } \
1529  \
1530  mask = 0x80; \
1531  value = 0; \
1532  if (x >= w) \
1533  continue; \
1534  \
1535  for (; x < w; x++) { \
1536  if (*ptr++ != bg) { \
1537  value |= mask; \
1538  } \
1539  mask >>= 1; \
1540  } \
1541  *buf++ = (uint8_t)value; \
1542  } \
1543 }
1544 
1548 
1549 
1550 /*
1551  * JPEG compression stuff.
1552  */
1553 
1554 static rfbBool
1555 SendJpegRect(rfbClientPtr cl, int x, int y, int w, int h, int quality)
1556 {
1557  unsigned char *srcbuf;
1558  int ps = cl->screen->serverFormat.bitsPerPixel / 8;
1559  int subsamp = subsampLevel2tjsubsamp[subsampLevel];
1560  unsigned long size = 0;
1561  int flags = 0, pitch;
1562  unsigned char *tmpbuf = NULL;
1563 
1564  if (cl->screen->serverFormat.bitsPerPixel == 8)
1565  return SendFullColorRect(cl, x, y, w, h);
1566 
1567  if (ps < 2) {
1568  rfbLog("Error: JPEG requires 16-bit, 24-bit, or 32-bit pixel format.\n");
1569  return 0;
1570  }
1571  if (!j) {
1572  if ((j = tjInitCompress()) == NULL) {
1573  rfbLog("JPEG Error: %s\n", tjGetErrorStr());
1574  return 0;
1575  }
1576  }
1577 
1578  if (tightAfterBufSize < TJBUFSIZE(w, h)) {
1579  if (tightAfterBuf == NULL)
1580  tightAfterBuf = (char *)malloc(TJBUFSIZE(w, h));
1581  else
1582  tightAfterBuf = (char *)realloc(tightAfterBuf,
1583  TJBUFSIZE(w, h));
1584  if (!tightAfterBuf) {
1585  rfbLog("Memory allocation failure!\n");
1586  return 0;
1587  }
1588  tightAfterBufSize = TJBUFSIZE(w, h);
1589  }
1590 
1591  if (ps == 2) {
1592  uint16_t *srcptr, pix;
1593  unsigned char *dst;
1594  int inRed, inGreen, inBlue, i, j;
1595 
1596  if((tmpbuf = (unsigned char *)malloc(w * h * 3)) == NULL)
1597  rfbLog("Memory allocation failure!\n");
1598  srcptr = (uint16_t *)&cl->scaledScreen->frameBuffer
1599  [y * cl->scaledScreen->paddedWidthInBytes + x * ps];
1600  dst = tmpbuf;
1601  for(j = 0; j < h; j++) {
1602  uint16_t *srcptr2 = srcptr;
1603  unsigned char *dst2 = dst;
1604  for (i = 0; i < w; i++) {
1605  pix = *srcptr2++;
1606  inRed = (int) (pix >> cl->screen->serverFormat.redShift
1607  & cl->screen->serverFormat.redMax);
1608  inGreen = (int) (pix >> cl->screen->serverFormat.greenShift
1609  & cl->screen->serverFormat.greenMax);
1610  inBlue = (int) (pix >> cl->screen->serverFormat.blueShift
1611  & cl->screen->serverFormat.blueMax);
1612  *dst2++ = (uint8_t)((inRed * 255
1613  + cl->screen->serverFormat.redMax / 2)
1614  / cl->screen->serverFormat.redMax);
1615  *dst2++ = (uint8_t)((inGreen * 255
1616  + cl->screen->serverFormat.greenMax / 2)
1617  / cl->screen->serverFormat.greenMax);
1618  *dst2++ = (uint8_t)((inBlue * 255
1619  + cl->screen->serverFormat.blueMax / 2)
1620  / cl->screen->serverFormat.blueMax);
1621  }
1622  srcptr += cl->scaledScreen->paddedWidthInBytes / ps;
1623  dst += w * 3;
1624  }
1625  srcbuf = tmpbuf;
1626  pitch = w * 3;
1627  ps = 3;
1628  } else {
1629  if (cl->screen->serverFormat.bigEndian && ps == 4)
1630  flags |= TJ_ALPHAFIRST;
1631  if (cl->screen->serverFormat.redShift == 16
1632  && cl->screen->serverFormat.blueShift == 0)
1633  flags |= TJ_BGR;
1634  if (cl->screen->serverFormat.bigEndian)
1635  flags ^= TJ_BGR;
1636  pitch = cl->scaledScreen->paddedWidthInBytes;
1637  srcbuf = (unsigned char *)&cl->scaledScreen->frameBuffer
1638  [y * pitch + x * ps];
1639  }
1640 
1641  if (tjCompress(j, srcbuf, w, pitch, h, ps, (unsigned char *)tightAfterBuf,
1642  &size, subsamp, quality, flags) == -1) {
1643  rfbLog("JPEG Error: %s\n", tjGetErrorStr());
1644  if (tmpbuf) {
1645  free(tmpbuf);
1646  tmpbuf = NULL;
1647  }
1648  return 0;
1649  }
1650 
1651  if (tmpbuf) {
1652  free(tmpbuf);
1653  tmpbuf = NULL;
1654  }
1655 
1656  if (cl->ublen + TIGHT_MIN_TO_COMPRESS + 1 > UPDATE_BUF_SIZE) {
1657  if (!rfbSendUpdateBuf(cl))
1658  return FALSE;
1659  }
1660 
1661  cl->updateBuf[cl->ublen++] = (char)(rfbTightJpeg << 4);
1662  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding, 1);
1663 
1664  return SendCompressedData(cl, tightAfterBuf, (int)size);
1665 }
1666 
1667 static void
1668 PrepareRowForImg(rfbClientPtr cl,
1669  uint8_t *dst,
1670  int x,
1671  int y,
1672  int count)
1673 {
1674  if (cl->screen->serverFormat.bitsPerPixel == 32) {
1675  if ( cl->screen->serverFormat.redMax == 0xFF &&
1676  cl->screen->serverFormat.greenMax == 0xFF &&
1677  cl->screen->serverFormat.blueMax == 0xFF ) {
1678  PrepareRowForImg24(cl, dst, x, y, count);
1679  } else {
1680  PrepareRowForImg32(cl, dst, x, y, count);
1681  }
1682  } else {
1683  /* 16 bpp assumed. */
1684  PrepareRowForImg16(cl, dst, x, y, count);
1685  }
1686 }
1687 
1688 static void
1689 PrepareRowForImg24(rfbClientPtr cl,
1690  uint8_t *dst,
1691  int x,
1692  int y,
1693  int count)
1694 {
1695  uint32_t *fbptr;
1696  uint32_t pix;
1697 
1698  fbptr = (uint32_t *)
1699  &cl->scaledScreen->frameBuffer[y * cl->scaledScreen->paddedWidthInBytes + x * 4];
1700 
1701  while (count--) {
1702  pix = *fbptr++;
1703  *dst++ = (uint8_t)(pix >> cl->screen->serverFormat.redShift);
1704  *dst++ = (uint8_t)(pix >> cl->screen->serverFormat.greenShift);
1705  *dst++ = (uint8_t)(pix >> cl->screen->serverFormat.blueShift);
1706  }
1707 }
1708 
1709 #define DEFINE_JPEG_GET_ROW_FUNCTION(bpp) \
1710  \
1711 static void \
1712 PrepareRowForImg##bpp(rfbClientPtr cl, uint8_t *dst, int x, int y, int count) { \
1713  uint##bpp##_t *fbptr; \
1714  uint##bpp##_t pix; \
1715  int inRed, inGreen, inBlue; \
1716  \
1717  fbptr = (uint##bpp##_t *) \
1718  &cl->scaledScreen->frameBuffer[y * cl->scaledScreen->paddedWidthInBytes + \
1719  x * (bpp / 8)]; \
1720  \
1721  while (count--) { \
1722  pix = *fbptr++; \
1723  \
1724  inRed = (int) \
1725  (pix >> cl->screen->serverFormat.redShift & cl->screen->serverFormat.redMax); \
1726  inGreen = (int) \
1727  (pix >> cl->screen->serverFormat.greenShift & cl->screen->serverFormat.greenMax); \
1728  inBlue = (int) \
1729  (pix >> cl->screen->serverFormat.blueShift & cl->screen->serverFormat.blueMax); \
1730  \
1731  *dst++ = (uint8_t)((inRed * 255 + cl->screen->serverFormat.redMax / 2) / \
1732  cl->screen->serverFormat.redMax); \
1733  *dst++ = (uint8_t)((inGreen * 255 + cl->screen->serverFormat.greenMax / 2) / \
1734  cl->screen->serverFormat.greenMax); \
1735  *dst++ = (uint8_t)((inBlue * 255 + cl->screen->serverFormat.blueMax / 2) / \
1736  cl->screen->serverFormat.blueMax); \
1737  } \
1738 }
1739 
1742 
1743 /*
1744  * PNG compression stuff.
1745  */
1746 
1747 #ifdef LIBVNCSERVER_HAVE_LIBPNG
1748 
1749 static TLS int pngDstDataLen = 0;
1750 
1751 static rfbBool CanSendPngRect(rfbClientPtr cl, int w, int h) {
1752  if (cl->tightEncoding != rfbEncodingTightPng) {
1753  return FALSE;
1754  }
1755 
1756  if ( cl->screen->serverFormat.bitsPerPixel == 8 ||
1757  cl->format.bitsPerPixel == 8) {
1758  return FALSE;
1759  }
1760 
1761  return TRUE;
1762 }
1763 
1764 static void pngWriteData(png_structp png_ptr, png_bytep data,
1765  png_size_t length)
1766 {
1767 #if 0
1768  rfbClientPtr cl = png_get_io_ptr(png_ptr);
1769 
1770  buffer_reserve(&vs->tight.png, vs->tight.png.offset + length);
1771  memcpy(vs->tight.png.buffer + vs->tight.png.offset, data, length);
1772 #endif
1773  memcpy(tightAfterBuf + pngDstDataLen, data, length);
1774 
1775  pngDstDataLen += length;
1776 }
1777 
1778 static void pngFlushData(png_structp png_ptr)
1779 {
1780 }
1781 
1782 
1783 static void *pngMalloc(png_structp png_ptr, png_size_t size)
1784 {
1785  return malloc(size);
1786 }
1787 
1788 static void pngFree(png_structp png_ptr, png_voidp ptr)
1789 {
1790  free(ptr);
1791 }
1792 
1793 static rfbBool SendPngRect(rfbClientPtr cl, int x, int y, int w, int h) {
1794  /* rfbLog(">> SendPngRect x:%d, y:%d, w:%d, h:%d\n", x, y, w, h); */
1795 
1796  png_byte color_type;
1797  png_structp png_ptr;
1798  png_infop info_ptr;
1799  png_colorp png_palette = NULL;
1800  int level = tightPngConf[cl->tightCompressLevel].png_zlib_level;
1801  int filters = tightPngConf[cl->tightCompressLevel].png_filters;
1802  uint8_t *buf;
1803  int dy;
1804 
1805  pngDstDataLen = 0;
1806 
1807  png_ptr = png_create_write_struct_2(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL,
1808  NULL, pngMalloc, pngFree);
1809 
1810  if (png_ptr == NULL)
1811  return FALSE;
1812 
1813  info_ptr = png_create_info_struct(png_ptr);
1814 
1815  if (info_ptr == NULL) {
1816  png_destroy_write_struct(&png_ptr, NULL);
1817  return FALSE;
1818  }
1819 
1820  png_set_write_fn(png_ptr, (void *) cl, pngWriteData, pngFlushData);
1821  png_set_compression_level(png_ptr, level);
1822  png_set_filter(png_ptr, PNG_FILTER_TYPE_DEFAULT, filters);
1823 
1824 #if 0
1825  /* TODO: */
1826  if (palette) {
1827  color_type = PNG_COLOR_TYPE_PALETTE;
1828  } else {
1829  color_type = PNG_COLOR_TYPE_RGB;
1830  }
1831 #else
1832  color_type = PNG_COLOR_TYPE_RGB;
1833 #endif
1834  png_set_IHDR(png_ptr, info_ptr, w, h,
1835  8, color_type, PNG_INTERLACE_NONE,
1836  PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT);
1837 
1838 #if 0
1839  if (color_type == PNG_COLOR_TYPE_PALETTE) {
1840  struct palette_cb_priv priv;
1841 
1842  png_palette = pngMalloc(png_ptr, sizeof(*png_palette) *
1843  palette_size(palette));
1844 
1845  priv.vs = vs;
1846  priv.png_palette = png_palette;
1847  palette_iter(palette, write_png_palette, &priv);
1848 
1849  png_set_PLTE(png_ptr, info_ptr, png_palette, palette_size(palette));
1850 
1851  offset = vs->tight.tight.offset;
1852  if (vs->clientds.pf.bytes_per_pixel == 4) {
1853  tight_encode_indexed_rect32(vs->tight.tight.buffer, w * h, palette);
1854  } else {
1855  tight_encode_indexed_rect16(vs->tight.tight.buffer, w * h, palette);
1856  }
1857  }
1858 
1859  buffer_reserve(&vs->tight.png, 2048);
1860 #endif
1861 
1862  png_write_info(png_ptr, info_ptr);
1863  buf = malloc(w * 3);
1864  for (dy = 0; dy < h; dy++)
1865  {
1866 #if 0
1867  if (color_type == PNG_COLOR_TYPE_PALETTE) {
1868  memcpy(buf, vs->tight.tight.buffer + (dy * w), w);
1869  } else {
1870  PrepareRowForImg(cl, buf, x, y + dy, w);
1871  }
1872 #else
1873  PrepareRowForImg(cl, buf, x, y + dy, w);
1874 #endif
1875  png_write_row(png_ptr, buf);
1876  }
1877  free(buf);
1878 
1879  png_write_end(png_ptr, NULL);
1880 
1881  if (color_type == PNG_COLOR_TYPE_PALETTE) {
1882  pngFree(png_ptr, png_palette);
1883  }
1884 
1885  png_destroy_write_struct(&png_ptr, &info_ptr);
1886 
1887  /* done v */
1888 
1889  if (cl->ublen + TIGHT_MIN_TO_COMPRESS + 1 > UPDATE_BUF_SIZE) {
1890  if (!rfbSendUpdateBuf(cl))
1891  return FALSE;
1892  }
1893 
1894  cl->updateBuf[cl->ublen++] = (char)(rfbTightPng << 4);
1895  rfbStatRecordEncodingSentAdd(cl, cl->tightEncoding, 1);
1896 
1897  /* rfbLog("<< SendPngRect\n"); */
1898  return SendCompressedData(cl, tightAfterBuf, pngDstDataLen);
1899 }
1900 #endif