xref: /illumos-gate/usr/src/cmd/bhyve/rfb.c (revision 26624470)
1bf21cd93STycho Nightingale /*-
24c87aefeSPatrick Mooney  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
34c87aefeSPatrick Mooney  *
4bf21cd93STycho Nightingale  * Copyright (c) 2015 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
54c87aefeSPatrick Mooney  * Copyright (c) 2015 Leon Dang
6*26624470SAndy Fiddaman  * Copyright 2020 Joyent, Inc.
7bf21cd93STycho Nightingale  * All rights reserved.
8bf21cd93STycho Nightingale  *
9bf21cd93STycho Nightingale  * Redistribution and use in source and binary forms, with or without
10bf21cd93STycho Nightingale  * modification, are permitted provided that the following conditions
11bf21cd93STycho Nightingale  * are met:
12bf21cd93STycho Nightingale  * 1. Redistributions of source code must retain the above copyright
13bf21cd93STycho Nightingale  *    notice, this list of conditions and the following disclaimer.
14bf21cd93STycho Nightingale  * 2. Redistributions in binary form must reproduce the above copyright
15bf21cd93STycho Nightingale  *    notice, this list of conditions and the following disclaimer in the
16bf21cd93STycho Nightingale  *    documentation and/or other materials provided with the distribution.
17bf21cd93STycho Nightingale  *
18bf21cd93STycho Nightingale  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND
19bf21cd93STycho Nightingale  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20bf21cd93STycho Nightingale  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21bf21cd93STycho Nightingale  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22bf21cd93STycho Nightingale  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23bf21cd93STycho Nightingale  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24bf21cd93STycho Nightingale  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25bf21cd93STycho Nightingale  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26bf21cd93STycho Nightingale  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27bf21cd93STycho Nightingale  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28bf21cd93STycho Nightingale  * SUCH DAMAGE.
29bf21cd93STycho Nightingale  */
30bf21cd93STycho Nightingale 
31bf21cd93STycho Nightingale #include <sys/cdefs.h>
32bf21cd93STycho Nightingale __FBSDID("$FreeBSD$");
33bf21cd93STycho Nightingale 
344c87aefeSPatrick Mooney #include <sys/param.h>
354c87aefeSPatrick Mooney #ifndef WITHOUT_CAPSICUM
364c87aefeSPatrick Mooney #include <sys/capsicum.h>
374c87aefeSPatrick Mooney #endif
384c87aefeSPatrick Mooney #include <sys/endian.h>
39bf21cd93STycho Nightingale #include <sys/socket.h>
404c87aefeSPatrick Mooney #include <sys/select.h>
414c87aefeSPatrick Mooney #include <sys/time.h>
424c87aefeSPatrick Mooney #include <arpa/inet.h>
43dc8050e8SMarko Kiiskila #include <stdatomic.h>
444c87aefeSPatrick Mooney #include <machine/cpufunc.h>
454c87aefeSPatrick Mooney #include <machine/specialreg.h>
46bf21cd93STycho Nightingale #include <netinet/in.h>
474c87aefeSPatrick Mooney #include <netdb.h>
48bf21cd93STycho Nightingale 
49bf21cd93STycho Nightingale #include <assert.h>
504c87aefeSPatrick Mooney #ifndef WITHOUT_CAPSICUM
514c87aefeSPatrick Mooney #include <capsicum_helpers.h>
524c87aefeSPatrick Mooney #endif
534c87aefeSPatrick Mooney #include <err.h>
544c87aefeSPatrick Mooney #include <errno.h>
55bf21cd93STycho Nightingale #include <pthread.h>
564c87aefeSPatrick Mooney #include <pthread_np.h>
57bf21cd93STycho Nightingale #include <signal.h>
58bf21cd93STycho Nightingale #include <stdbool.h>
59bf21cd93STycho Nightingale #include <stdlib.h>
60bf21cd93STycho Nightingale #include <stdio.h>
61bf21cd93STycho Nightingale #include <string.h>
624c87aefeSPatrick Mooney #include <sysexits.h>
63bf21cd93STycho Nightingale #include <unistd.h>
64bf21cd93STycho Nightingale 
654c87aefeSPatrick Mooney #include <zlib.h>
664c87aefeSPatrick Mooney 
674c87aefeSPatrick Mooney #ifndef __FreeBSD__
684c87aefeSPatrick Mooney #include <sys/debug.h>
694c87aefeSPatrick Mooney #endif
704c87aefeSPatrick Mooney 
71bf21cd93STycho Nightingale #include "bhyvegc.h"
72154972afSPatrick Mooney #include "debug.h"
73bf21cd93STycho Nightingale #include "console.h"
74bf21cd93STycho Nightingale #include "rfb.h"
754c87aefeSPatrick Mooney #include "sockstream.h"
764c87aefeSPatrick Mooney 
774c87aefeSPatrick Mooney #ifndef NO_OPENSSL
784c87aefeSPatrick Mooney #include <openssl/des.h>
794c87aefeSPatrick Mooney #endif
804c87aefeSPatrick Mooney 
81dc8050e8SMarko Kiiskila /* Delays in microseconds */
82dc8050e8SMarko Kiiskila #define	CFD_SEL_DELAY	10000
83dc8050e8SMarko Kiiskila #define	SCREEN_REFRESH_DELAY	33300	/* 30Hz */
84dc8050e8SMarko Kiiskila #define	SCREEN_POLL_DELAY	(SCREEN_REFRESH_DELAY / 2)
85dc8050e8SMarko Kiiskila 
864c87aefeSPatrick Mooney static int rfb_debug = 0;
87154972afSPatrick Mooney #define	DPRINTF(params) if (rfb_debug) PRINTLN params
88154972afSPatrick Mooney #define	WPRINTF(params) PRINTLN params
894c87aefeSPatrick Mooney 
90154972afSPatrick Mooney #define VERSION_LENGTH	12
914c87aefeSPatrick Mooney #define AUTH_LENGTH	16
924c87aefeSPatrick Mooney #define PASSWD_LENGTH	8
934c87aefeSPatrick Mooney 
94dc8050e8SMarko Kiiskila /* Protocol versions */
95dc8050e8SMarko Kiiskila #define CVERS_3_3	'3'
96dc8050e8SMarko Kiiskila #define CVERS_3_7	'7'
97dc8050e8SMarko Kiiskila #define CVERS_3_8	'8'
98dc8050e8SMarko Kiiskila 
99dc8050e8SMarko Kiiskila /* Client-to-server msg types */
100dc8050e8SMarko Kiiskila #define CS_SET_PIXEL_FORMAT	0
101dc8050e8SMarko Kiiskila #define CS_SET_ENCODINGS	2
102dc8050e8SMarko Kiiskila #define CS_UPDATE_MSG		3
103dc8050e8SMarko Kiiskila #define CS_KEY_EVENT		4
104dc8050e8SMarko Kiiskila #define CS_POINTER_EVENT	5
105dc8050e8SMarko Kiiskila #define CS_CUT_TEXT		6
106dc8050e8SMarko Kiiskila 
1074c87aefeSPatrick Mooney #define SECURITY_TYPE_NONE	1
1084c87aefeSPatrick Mooney #define SECURITY_TYPE_VNC_AUTH	2
1094c87aefeSPatrick Mooney 
1104c87aefeSPatrick Mooney #define AUTH_FAILED_UNAUTH	1
1114c87aefeSPatrick Mooney #define AUTH_FAILED_ERROR	2
112bf21cd93STycho Nightingale 
113bf21cd93STycho Nightingale struct rfb_softc {
114bf21cd93STycho Nightingale 	int		sfd;
115bf21cd93STycho Nightingale 	pthread_t	tid;
116bf21cd93STycho Nightingale 
1174c87aefeSPatrick Mooney 	int		cfd;
1184c87aefeSPatrick Mooney 
119bf21cd93STycho Nightingale 	int		width, height;
120bf21cd93STycho Nightingale 
1214c87aefeSPatrick Mooney 	char		*password;
1224c87aefeSPatrick Mooney 
123dc8050e8SMarko Kiiskila 	bool		enc_raw_ok;
124dc8050e8SMarko Kiiskila 	bool		enc_zlib_ok;
125dc8050e8SMarko Kiiskila 	bool		enc_resize_ok;
1264c87aefeSPatrick Mooney 
1274c87aefeSPatrick Mooney 	z_stream	zstream;
1284c87aefeSPatrick Mooney 	uint8_t		*zbuf;
1294c87aefeSPatrick Mooney 	int		zbuflen;
1304c87aefeSPatrick Mooney 
1314c87aefeSPatrick Mooney 	int		conn_wait;
132dc8050e8SMarko Kiiskila 	int		wrcount;
133dc8050e8SMarko Kiiskila 
134dc8050e8SMarko Kiiskila 	atomic_bool	sending;
135dc8050e8SMarko Kiiskila 	atomic_bool	pending;
136dc8050e8SMarko Kiiskila 	atomic_bool	update_all;
137dc8050e8SMarko Kiiskila 	atomic_bool	input_detected;
138dc8050e8SMarko Kiiskila 
1394c87aefeSPatrick Mooney 	pthread_mutex_t mtx;
1404c87aefeSPatrick Mooney 	pthread_cond_t  cond;
1414c87aefeSPatrick Mooney 
1424c87aefeSPatrick Mooney 	int		hw_crc;
1434c87aefeSPatrick Mooney 	uint32_t	*crc;		/* WxH crc cells */
1444c87aefeSPatrick Mooney 	uint32_t	*crc_tmp;	/* buffer to store single crc row */
1454c87aefeSPatrick Mooney 	int		crc_width, crc_height;
146bf21cd93STycho Nightingale };
147bf21cd93STycho Nightingale 
148bf21cd93STycho Nightingale struct rfb_pixfmt {
149bf21cd93STycho Nightingale 	uint8_t		bpp;
150bf21cd93STycho Nightingale 	uint8_t		depth;
151bf21cd93STycho Nightingale 	uint8_t		bigendian;
152bf21cd93STycho Nightingale 	uint8_t		truecolor;
153bf21cd93STycho Nightingale 	uint16_t	red_max;
154bf21cd93STycho Nightingale 	uint16_t	green_max;
155bf21cd93STycho Nightingale 	uint16_t	blue_max;
156bf21cd93STycho Nightingale 	uint8_t		red_shift;
157bf21cd93STycho Nightingale 	uint8_t		green_shift;
158bf21cd93STycho Nightingale 	uint8_t		blue_shift;
159bf21cd93STycho Nightingale 	uint8_t		pad[3];
160bf21cd93STycho Nightingale };
161bf21cd93STycho Nightingale 
162bf21cd93STycho Nightingale struct rfb_srvr_info {
163bf21cd93STycho Nightingale 	uint16_t		width;
164bf21cd93STycho Nightingale 	uint16_t		height;
165bf21cd93STycho Nightingale 	struct rfb_pixfmt	pixfmt;
166bf21cd93STycho Nightingale 	uint32_t		namelen;
167bf21cd93STycho Nightingale };
168bf21cd93STycho Nightingale 
169bf21cd93STycho Nightingale struct rfb_pixfmt_msg {
170bf21cd93STycho Nightingale 	uint8_t			type;
171bf21cd93STycho Nightingale 	uint8_t			pad[3];
172bf21cd93STycho Nightingale 	struct rfb_pixfmt	pixfmt;
173bf21cd93STycho Nightingale };
174bf21cd93STycho Nightingale 
175bf21cd93STycho Nightingale #define	RFB_ENCODING_RAW		0
1764c87aefeSPatrick Mooney #define	RFB_ENCODING_ZLIB		6
177bf21cd93STycho Nightingale #define	RFB_ENCODING_RESIZE		-223
178bf21cd93STycho Nightingale 
1794c87aefeSPatrick Mooney #define	RFB_MAX_WIDTH			2000
1804c87aefeSPatrick Mooney #define	RFB_MAX_HEIGHT			1200
1814c87aefeSPatrick Mooney #define	RFB_ZLIB_BUFSZ			RFB_MAX_WIDTH*RFB_MAX_HEIGHT*4
1824c87aefeSPatrick Mooney 
1834c87aefeSPatrick Mooney /* percentage changes to screen before sending the entire screen */
1844c87aefeSPatrick Mooney #define	RFB_SEND_ALL_THRESH		25
1854c87aefeSPatrick Mooney 
186bf21cd93STycho Nightingale struct rfb_enc_msg {
187bf21cd93STycho Nightingale 	uint8_t		type;
188bf21cd93STycho Nightingale 	uint8_t		pad;
189bf21cd93STycho Nightingale 	uint16_t	numencs;
190bf21cd93STycho Nightingale };
191bf21cd93STycho Nightingale 
192bf21cd93STycho Nightingale struct rfb_updt_msg {
193bf21cd93STycho Nightingale 	uint8_t		type;
194bf21cd93STycho Nightingale 	uint8_t		incremental;
195bf21cd93STycho Nightingale 	uint16_t	x;
196bf21cd93STycho Nightingale 	uint16_t	y;
197bf21cd93STycho Nightingale 	uint16_t	width;
198bf21cd93STycho Nightingale 	uint16_t	height;
199bf21cd93STycho Nightingale };
200bf21cd93STycho Nightingale 
201bf21cd93STycho Nightingale struct rfb_key_msg {
202bf21cd93STycho Nightingale 	uint8_t		type;
203bf21cd93STycho Nightingale 	uint8_t		down;
204bf21cd93STycho Nightingale 	uint16_t	pad;
205bf21cd93STycho Nightingale 	uint32_t	code;
206bf21cd93STycho Nightingale };
207bf21cd93STycho Nightingale 
208bf21cd93STycho Nightingale struct rfb_ptr_msg {
209bf21cd93STycho Nightingale 	uint8_t		type;
210bf21cd93STycho Nightingale 	uint8_t		button;
211bf21cd93STycho Nightingale 	uint16_t	x;
212bf21cd93STycho Nightingale 	uint16_t	y;
213bf21cd93STycho Nightingale };
214bf21cd93STycho Nightingale 
215bf21cd93STycho Nightingale struct rfb_srvr_updt_msg {
216bf21cd93STycho Nightingale 	uint8_t		type;
217bf21cd93STycho Nightingale 	uint8_t		pad;
218bf21cd93STycho Nightingale 	uint16_t	numrects;
219bf21cd93STycho Nightingale };
220bf21cd93STycho Nightingale 
221bf21cd93STycho Nightingale struct rfb_srvr_rect_hdr {
222bf21cd93STycho Nightingale 	uint16_t	x;
223bf21cd93STycho Nightingale 	uint16_t	y;
224bf21cd93STycho Nightingale 	uint16_t	width;
225bf21cd93STycho Nightingale 	uint16_t	height;
226bf21cd93STycho Nightingale 	uint32_t	encoding;
227bf21cd93STycho Nightingale };
228bf21cd93STycho Nightingale 
2294c87aefeSPatrick Mooney struct rfb_cuttext_msg {
2304c87aefeSPatrick Mooney 	uint8_t		type;
2314c87aefeSPatrick Mooney 	uint8_t		padding[3];
2324c87aefeSPatrick Mooney 	uint32_t	length;
2334c87aefeSPatrick Mooney };
2344c87aefeSPatrick Mooney 
235bf21cd93STycho Nightingale static void
rfb_send_server_init_msg(int cfd)236bf21cd93STycho Nightingale rfb_send_server_init_msg(int cfd)
237bf21cd93STycho Nightingale {
238bf21cd93STycho Nightingale 	struct bhyvegc_image *gc_image;
239bf21cd93STycho Nightingale 	struct rfb_srvr_info sinfo;
240bf21cd93STycho Nightingale 
241bf21cd93STycho Nightingale 	gc_image = console_get_image();
242bf21cd93STycho Nightingale 
2434c87aefeSPatrick Mooney 	sinfo.width = htons(gc_image->width);
2444c87aefeSPatrick Mooney 	sinfo.height = htons(gc_image->height);
245bf21cd93STycho Nightingale 	sinfo.pixfmt.bpp = 32;
246bf21cd93STycho Nightingale 	sinfo.pixfmt.depth = 32;
247bf21cd93STycho Nightingale 	sinfo.pixfmt.bigendian = 0;
248bf21cd93STycho Nightingale 	sinfo.pixfmt.truecolor = 1;
2494c87aefeSPatrick Mooney 	sinfo.pixfmt.red_max = htons(255);
2504c87aefeSPatrick Mooney 	sinfo.pixfmt.green_max = htons(255);
2514c87aefeSPatrick Mooney 	sinfo.pixfmt.blue_max = htons(255);
252bf21cd93STycho Nightingale 	sinfo.pixfmt.red_shift = 16;
253bf21cd93STycho Nightingale 	sinfo.pixfmt.green_shift = 8;
254bf21cd93STycho Nightingale 	sinfo.pixfmt.blue_shift = 0;
255dc8050e8SMarko Kiiskila 	sinfo.pixfmt.pad[0] = 0;
256dc8050e8SMarko Kiiskila 	sinfo.pixfmt.pad[1] = 0;
257dc8050e8SMarko Kiiskila 	sinfo.pixfmt.pad[2] = 0;
2584c87aefeSPatrick Mooney 	sinfo.namelen = htonl(strlen("bhyve"));
2594c87aefeSPatrick Mooney 	(void)stream_write(cfd, &sinfo, sizeof(sinfo));
2604c87aefeSPatrick Mooney 	(void)stream_write(cfd, "bhyve", strlen("bhyve"));
261bf21cd93STycho Nightingale }
262bf21cd93STycho Nightingale 
263bf21cd93STycho Nightingale static void
rfb_send_resize_update_msg(struct rfb_softc * rc,int cfd)264bf21cd93STycho Nightingale rfb_send_resize_update_msg(struct rfb_softc *rc, int cfd)
265bf21cd93STycho Nightingale {
266bf21cd93STycho Nightingale 	struct rfb_srvr_updt_msg supdt_msg;
2674c87aefeSPatrick Mooney 	struct rfb_srvr_rect_hdr srect_hdr;
268bf21cd93STycho Nightingale 
269bf21cd93STycho Nightingale 	/* Number of rectangles: 1 */
270bf21cd93STycho Nightingale 	supdt_msg.type = 0;
271bf21cd93STycho Nightingale 	supdt_msg.pad = 0;
2724c87aefeSPatrick Mooney 	supdt_msg.numrects = htons(1);
2734c87aefeSPatrick Mooney 	stream_write(cfd, &supdt_msg, sizeof(struct rfb_srvr_updt_msg));
274bf21cd93STycho Nightingale 
275bf21cd93STycho Nightingale 	/* Rectangle header */
2764c87aefeSPatrick Mooney 	srect_hdr.x = htons(0);
2774c87aefeSPatrick Mooney 	srect_hdr.y = htons(0);
2784c87aefeSPatrick Mooney 	srect_hdr.width = htons(rc->width);
2794c87aefeSPatrick Mooney 	srect_hdr.height = htons(rc->height);
2804c87aefeSPatrick Mooney 	srect_hdr.encoding = htonl(RFB_ENCODING_RESIZE);
2814c87aefeSPatrick Mooney 	stream_write(cfd, &srect_hdr, sizeof(struct rfb_srvr_rect_hdr));
282bf21cd93STycho Nightingale }
283bf21cd93STycho Nightingale 
284bf21cd93STycho Nightingale static void
rfb_recv_set_pixfmt_msg(struct rfb_softc * rc,int cfd)285bf21cd93STycho Nightingale rfb_recv_set_pixfmt_msg(struct rfb_softc *rc, int cfd)
286bf21cd93STycho Nightingale {
287bf21cd93STycho Nightingale 	struct rfb_pixfmt_msg pixfmt_msg;
288bf21cd93STycho Nightingale 
2894c87aefeSPatrick Mooney 	(void)stream_read(cfd, ((void *)&pixfmt_msg)+1, sizeof(pixfmt_msg)-1);
290bf21cd93STycho Nightingale }
291bf21cd93STycho Nightingale 
292bf21cd93STycho Nightingale 
293bf21cd93STycho Nightingale static void
rfb_recv_set_encodings_msg(struct rfb_softc * rc,int cfd)294bf21cd93STycho Nightingale rfb_recv_set_encodings_msg(struct rfb_softc *rc, int cfd)
295bf21cd93STycho Nightingale {
296bf21cd93STycho Nightingale 	struct rfb_enc_msg enc_msg;
2974c87aefeSPatrick Mooney 	int i;
298bf21cd93STycho Nightingale 	uint32_t encoding;
299bf21cd93STycho Nightingale 
3004c87aefeSPatrick Mooney 	(void)stream_read(cfd, ((void *)&enc_msg)+1, sizeof(enc_msg)-1);
301bf21cd93STycho Nightingale 
3024c87aefeSPatrick Mooney 	for (i = 0; i < htons(enc_msg.numencs); i++) {
3034c87aefeSPatrick Mooney 		(void)stream_read(cfd, &encoding, sizeof(encoding));
3044c87aefeSPatrick Mooney 		switch (htonl(encoding)) {
305bf21cd93STycho Nightingale 		case RFB_ENCODING_RAW:
306bf21cd93STycho Nightingale 			rc->enc_raw_ok = true;
307bf21cd93STycho Nightingale 			break;
3084c87aefeSPatrick Mooney 		case RFB_ENCODING_ZLIB:
30984659b24SMichael Zeller 			if (!rc->enc_zlib_ok) {
31084659b24SMichael Zeller 				deflateInit(&rc->zstream, Z_BEST_SPEED);
31184659b24SMichael Zeller 				rc->enc_zlib_ok = true;
31284659b24SMichael Zeller 			}
3134c87aefeSPatrick Mooney 			break;
314bf21cd93STycho Nightingale 		case RFB_ENCODING_RESIZE:
315bf21cd93STycho Nightingale 			rc->enc_resize_ok = true;
316bf21cd93STycho Nightingale 			break;
317bf21cd93STycho Nightingale 		}
318bf21cd93STycho Nightingale 	}
319bf21cd93STycho Nightingale }
320bf21cd93STycho Nightingale 
3214c87aefeSPatrick Mooney /*
3224c87aefeSPatrick Mooney  * Calculate CRC32 using SSE4.2; Intel or AMD Bulldozer+ CPUs only
3234c87aefeSPatrick Mooney  */
3244c87aefeSPatrick Mooney static __inline uint32_t
fast_crc32(void * buf,int len,uint32_t crcval)3254c87aefeSPatrick Mooney fast_crc32(void *buf, int len, uint32_t crcval)
3264c87aefeSPatrick Mooney {
3274c87aefeSPatrick Mooney 	uint32_t q = len / sizeof(uint32_t);
3284c87aefeSPatrick Mooney 	uint32_t *p = (uint32_t *)buf;
3294c87aefeSPatrick Mooney 
3304c87aefeSPatrick Mooney 	while (q--) {
3314c87aefeSPatrick Mooney 		asm volatile (
3324c87aefeSPatrick Mooney 			".byte 0xf2, 0xf, 0x38, 0xf1, 0xf1;"
3334c87aefeSPatrick Mooney 			:"=S" (crcval)
3344c87aefeSPatrick Mooney 			:"0" (crcval), "c" (*p)
3354c87aefeSPatrick Mooney 		);
3364c87aefeSPatrick Mooney 		p++;
3374c87aefeSPatrick Mooney 	}
3384c87aefeSPatrick Mooney 
3394c87aefeSPatrick Mooney 	return (crcval);
3404c87aefeSPatrick Mooney }
3414c87aefeSPatrick Mooney 
342dc8050e8SMarko Kiiskila static int
rfb_send_update_header(struct rfb_softc * rc,int cfd,int numrects)343dc8050e8SMarko Kiiskila rfb_send_update_header(struct rfb_softc *rc, int cfd, int numrects)
344dc8050e8SMarko Kiiskila {
345dc8050e8SMarko Kiiskila 	struct rfb_srvr_updt_msg supdt_msg;
346dc8050e8SMarko Kiiskila 
347dc8050e8SMarko Kiiskila 	supdt_msg.type = 0;
348dc8050e8SMarko Kiiskila 	supdt_msg.pad = 0;
349dc8050e8SMarko Kiiskila 	supdt_msg.numrects = htons(numrects);
350dc8050e8SMarko Kiiskila 
351dc8050e8SMarko Kiiskila 	return stream_write(cfd, &supdt_msg,
352dc8050e8SMarko Kiiskila 	    sizeof(struct rfb_srvr_updt_msg));
353dc8050e8SMarko Kiiskila }
3544c87aefeSPatrick Mooney 
3554c87aefeSPatrick Mooney static int
rfb_send_rect(struct rfb_softc * rc,int cfd,struct bhyvegc_image * gc,int x,int y,int w,int h)3564c87aefeSPatrick Mooney rfb_send_rect(struct rfb_softc *rc, int cfd, struct bhyvegc_image *gc,
3574c87aefeSPatrick Mooney               int x, int y, int w, int h)
3584c87aefeSPatrick Mooney {
3594c87aefeSPatrick Mooney 	struct rfb_srvr_rect_hdr srect_hdr;
3604c87aefeSPatrick Mooney 	unsigned long zlen;
3614c87aefeSPatrick Mooney 	ssize_t nwrite, total;
3624c87aefeSPatrick Mooney 	int err;
3634c87aefeSPatrick Mooney 	uint32_t *p;
3644c87aefeSPatrick Mooney 	uint8_t *zbufp;
3654c87aefeSPatrick Mooney 
3664c87aefeSPatrick Mooney 	/*
3674c87aefeSPatrick Mooney 	 * Send a single rectangle of the given x, y, w h dimensions.
3684c87aefeSPatrick Mooney 	 */
3694c87aefeSPatrick Mooney 
3704c87aefeSPatrick Mooney 	/* Rectangle header */
3714c87aefeSPatrick Mooney 	srect_hdr.x = htons(x);
3724c87aefeSPatrick Mooney 	srect_hdr.y = htons(y);
3734c87aefeSPatrick Mooney 	srect_hdr.width = htons(w);
3744c87aefeSPatrick Mooney 	srect_hdr.height = htons(h);
3754c87aefeSPatrick Mooney 
3764c87aefeSPatrick Mooney 	h = y + h;
3774c87aefeSPatrick Mooney 	w *= sizeof(uint32_t);
3784c87aefeSPatrick Mooney 	if (rc->enc_zlib_ok) {
3794c87aefeSPatrick Mooney 		zbufp = rc->zbuf;
3804c87aefeSPatrick Mooney 		rc->zstream.total_in = 0;
3814c87aefeSPatrick Mooney 		rc->zstream.total_out = 0;
3824c87aefeSPatrick Mooney 		for (p = &gc->data[y * gc->width + x]; y < h; y++) {
3834c87aefeSPatrick Mooney 			rc->zstream.next_in = (Bytef *)p;
3844c87aefeSPatrick Mooney 			rc->zstream.avail_in = w;
3854c87aefeSPatrick Mooney 			rc->zstream.next_out = (Bytef *)zbufp;
3864c87aefeSPatrick Mooney 			rc->zstream.avail_out = RFB_ZLIB_BUFSZ + 16 -
3874c87aefeSPatrick Mooney 			                        rc->zstream.total_out;
3884c87aefeSPatrick Mooney 			rc->zstream.data_type = Z_BINARY;
3894c87aefeSPatrick Mooney 
3904c87aefeSPatrick Mooney 			/* Compress with zlib */
3914c87aefeSPatrick Mooney 			err = deflate(&rc->zstream, Z_SYNC_FLUSH);
3924c87aefeSPatrick Mooney 			if (err != Z_OK) {
393154972afSPatrick Mooney 				WPRINTF(("zlib[rect] deflate err: %d", err));
3944c87aefeSPatrick Mooney 				rc->enc_zlib_ok = false;
3954c87aefeSPatrick Mooney 				deflateEnd(&rc->zstream);
3964c87aefeSPatrick Mooney 				goto doraw;
3974c87aefeSPatrick Mooney 			}
3984c87aefeSPatrick Mooney 			zbufp = rc->zbuf + rc->zstream.total_out;
3994c87aefeSPatrick Mooney 			p += gc->width;
4004c87aefeSPatrick Mooney 		}
4014c87aefeSPatrick Mooney 		srect_hdr.encoding = htonl(RFB_ENCODING_ZLIB);
4024c87aefeSPatrick Mooney 		nwrite = stream_write(cfd, &srect_hdr,
4034c87aefeSPatrick Mooney 		                      sizeof(struct rfb_srvr_rect_hdr));
4044c87aefeSPatrick Mooney 		if (nwrite <= 0)
4054c87aefeSPatrick Mooney 			return (nwrite);
4064c87aefeSPatrick Mooney 
4074c87aefeSPatrick Mooney 		zlen = htonl(rc->zstream.total_out);
4084c87aefeSPatrick Mooney 		nwrite = stream_write(cfd, &zlen, sizeof(uint32_t));
4094c87aefeSPatrick Mooney 		if (nwrite <= 0)
4104c87aefeSPatrick Mooney 			return (nwrite);
4114c87aefeSPatrick Mooney 		return (stream_write(cfd, rc->zbuf, rc->zstream.total_out));
4124c87aefeSPatrick Mooney 	}
4134c87aefeSPatrick Mooney 
4144c87aefeSPatrick Mooney doraw:
4154c87aefeSPatrick Mooney 
4164c87aefeSPatrick Mooney 	total = 0;
4174c87aefeSPatrick Mooney 	zbufp = rc->zbuf;
4184c87aefeSPatrick Mooney 	for (p = &gc->data[y * gc->width + x]; y < h; y++) {
4194c87aefeSPatrick Mooney 		memcpy(zbufp, p, w);
4204c87aefeSPatrick Mooney 		zbufp += w;
4214c87aefeSPatrick Mooney 		total += w;
4224c87aefeSPatrick Mooney 		p += gc->width;
4234c87aefeSPatrick Mooney 	}
4244c87aefeSPatrick Mooney 
4254c87aefeSPatrick Mooney 	srect_hdr.encoding = htonl(RFB_ENCODING_RAW);
4264c87aefeSPatrick Mooney 	nwrite = stream_write(cfd, &srect_hdr,
4274c87aefeSPatrick Mooney 	                      sizeof(struct rfb_srvr_rect_hdr));
4284c87aefeSPatrick Mooney 	if (nwrite <= 0)
4294c87aefeSPatrick Mooney 		return (nwrite);
4304c87aefeSPatrick Mooney 
4314c87aefeSPatrick Mooney 	total = stream_write(cfd, rc->zbuf, total);
4324c87aefeSPatrick Mooney 
4334c87aefeSPatrick Mooney 	return (total);
4344c87aefeSPatrick Mooney }
4354c87aefeSPatrick Mooney 
4364c87aefeSPatrick Mooney static int
rfb_send_all(struct rfb_softc * rc,int cfd,struct bhyvegc_image * gc)4374c87aefeSPatrick Mooney rfb_send_all(struct rfb_softc *rc, int cfd, struct bhyvegc_image *gc)
438bf21cd93STycho Nightingale {
439bf21cd93STycho Nightingale 	struct rfb_srvr_updt_msg supdt_msg;
440dc8050e8SMarko Kiiskila 	struct rfb_srvr_rect_hdr srect_hdr;
4414c87aefeSPatrick Mooney 	ssize_t nwrite;
4424c87aefeSPatrick Mooney 	unsigned long zlen;
4434c87aefeSPatrick Mooney 	int err;
4444c87aefeSPatrick Mooney 
4454c87aefeSPatrick Mooney 	/*
4464c87aefeSPatrick Mooney 	 * Send the whole thing
4474c87aefeSPatrick Mooney 	 */
448bf21cd93STycho Nightingale 
449bf21cd93STycho Nightingale 	/* Number of rectangles: 1 */
450bf21cd93STycho Nightingale 	supdt_msg.type = 0;
451bf21cd93STycho Nightingale 	supdt_msg.pad = 0;
4524c87aefeSPatrick Mooney 	supdt_msg.numrects = htons(1);
4534c87aefeSPatrick Mooney 	nwrite = stream_write(cfd, &supdt_msg,
4544c87aefeSPatrick Mooney 	                      sizeof(struct rfb_srvr_updt_msg));
4554c87aefeSPatrick Mooney 	if (nwrite <= 0)
4564c87aefeSPatrick Mooney 		return (nwrite);
457bf21cd93STycho Nightingale 
458bf21cd93STycho Nightingale 	/* Rectangle header */
4594c87aefeSPatrick Mooney 	srect_hdr.x = 0;
4604c87aefeSPatrick Mooney 	srect_hdr.y = 0;
4614c87aefeSPatrick Mooney 	srect_hdr.width = htons(gc->width);
4624c87aefeSPatrick Mooney 	srect_hdr.height = htons(gc->height);
4634c87aefeSPatrick Mooney 	if (rc->enc_zlib_ok) {
4644c87aefeSPatrick Mooney 		rc->zstream.next_in = (Bytef *)gc->data;
4654c87aefeSPatrick Mooney 		rc->zstream.avail_in = gc->width * gc->height *
4664c87aefeSPatrick Mooney 		                   sizeof(uint32_t);
4674c87aefeSPatrick Mooney 		rc->zstream.next_out = (Bytef *)rc->zbuf;
4684c87aefeSPatrick Mooney 		rc->zstream.avail_out = RFB_ZLIB_BUFSZ + 16;
4694c87aefeSPatrick Mooney 		rc->zstream.data_type = Z_BINARY;
4704c87aefeSPatrick Mooney 
4714c87aefeSPatrick Mooney 		rc->zstream.total_in = 0;
4724c87aefeSPatrick Mooney 		rc->zstream.total_out = 0;
4734c87aefeSPatrick Mooney 
4744c87aefeSPatrick Mooney 		/* Compress with zlib */
4754c87aefeSPatrick Mooney 		err = deflate(&rc->zstream, Z_SYNC_FLUSH);
4764c87aefeSPatrick Mooney 		if (err != Z_OK) {
477154972afSPatrick Mooney 			WPRINTF(("zlib deflate err: %d", err));
4784c87aefeSPatrick Mooney 			rc->enc_zlib_ok = false;
4794c87aefeSPatrick Mooney 			deflateEnd(&rc->zstream);
4804c87aefeSPatrick Mooney 			goto doraw;
4814c87aefeSPatrick Mooney 		}
4824c87aefeSPatrick Mooney 
4834c87aefeSPatrick Mooney 		srect_hdr.encoding = htonl(RFB_ENCODING_ZLIB);
4844c87aefeSPatrick Mooney 		nwrite = stream_write(cfd, &srect_hdr,
4854c87aefeSPatrick Mooney 		                      sizeof(struct rfb_srvr_rect_hdr));
4864c87aefeSPatrick Mooney 		if (nwrite <= 0)
4874c87aefeSPatrick Mooney 			return (nwrite);
4884c87aefeSPatrick Mooney 
4894c87aefeSPatrick Mooney 		zlen = htonl(rc->zstream.total_out);
4904c87aefeSPatrick Mooney 		nwrite = stream_write(cfd, &zlen, sizeof(uint32_t));
4914c87aefeSPatrick Mooney 		if (nwrite <= 0)
4924c87aefeSPatrick Mooney 			return (nwrite);
4934c87aefeSPatrick Mooney 		return (stream_write(cfd, rc->zbuf, rc->zstream.total_out));
4944c87aefeSPatrick Mooney 	}
4954c87aefeSPatrick Mooney 
4964c87aefeSPatrick Mooney doraw:
4974c87aefeSPatrick Mooney 	srect_hdr.encoding = htonl(RFB_ENCODING_RAW);
4984c87aefeSPatrick Mooney 	nwrite = stream_write(cfd, &srect_hdr,
4994c87aefeSPatrick Mooney 	                      sizeof(struct rfb_srvr_rect_hdr));
5004c87aefeSPatrick Mooney 	if (nwrite <= 0)
5014c87aefeSPatrick Mooney 		return (nwrite);
5024c87aefeSPatrick Mooney 
5034c87aefeSPatrick Mooney 	nwrite = stream_write(cfd, gc->data,
5044c87aefeSPatrick Mooney 	               gc->width * gc->height * sizeof(uint32_t));
5054c87aefeSPatrick Mooney 
5064c87aefeSPatrick Mooney 	return (nwrite);
507bf21cd93STycho Nightingale }
508bf21cd93STycho Nightingale 
5094c87aefeSPatrick Mooney #define	PIX_PER_CELL	32
5104c87aefeSPatrick Mooney #define	PIXCELL_SHIFT	5
5114c87aefeSPatrick Mooney #define	PIXCELL_MASK	0x1F
5124c87aefeSPatrick Mooney 
5134c87aefeSPatrick Mooney static int
rfb_send_screen(struct rfb_softc * rc,int cfd)514dc8050e8SMarko Kiiskila rfb_send_screen(struct rfb_softc *rc, int cfd)
5154c87aefeSPatrick Mooney {
5164c87aefeSPatrick Mooney 	struct bhyvegc_image *gc_image;
5174c87aefeSPatrick Mooney 	ssize_t nwrite;
5184c87aefeSPatrick Mooney 	int x, y;
5194c87aefeSPatrick Mooney 	int celly, cellwidth;
5204c87aefeSPatrick Mooney 	int xcells, ycells;
5214c87aefeSPatrick Mooney 	int w, h;
5224c87aefeSPatrick Mooney 	uint32_t *p;
5234c87aefeSPatrick Mooney 	int rem_x, rem_y;   /* remainder for resolutions not x32 pixels ratio */
5244c87aefeSPatrick Mooney 	int retval;
5254c87aefeSPatrick Mooney 	uint32_t *crc_p, *orig_crc;
5264c87aefeSPatrick Mooney 	int changes;
527