1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 #include <stdio.h>
26 #include <stdlib.h>
27 #include <string.h>
28 #include <strings.h>
29 #include <sys/types.h>
30 #include <errno.h>
31 #include <syslog.h>
32 #include <unistd.h>
33 #include <sys/types.h>
34 #include <sys/socket.h>
35 #include <sys/time.h>
36 #include <netinet/in.h>
37 #include <arpa/inet.h>
38 #include <netdb.h>
39 #include <sys/stat.h>
40 #include <sys/sdt.h>
41 #include <signal.h>
42 #include <fcntl.h>
43 #include <libnvpair.h>
44 #include <libstmf.h>
45 #include <door.h>
46 #include <pthread.h>
47 #include <libscf.h>
48 #include <locale.h>
49 #include <sys/stmf_ioctl.h>
50 #include <sys/pppt_ioctl.h>
51 #include <libstmfproxy.h>
52 
53 #define	PPPT_NODE	"/devices/pseudo/pppt@0:pppt"
54 #define	USAGE	"Usage: %s [-d][-f][-n nodeid] nodename\n" \
55 		"Note: nodename must be the same on both nodes\n"
56 
57 
58 /*
59  * static functions
60  */
61 static void daemonInit(void);
62 static void killHandler();
63 static int postMsg(uint_t nelem, uchar_t *aluaMsg);
64 
65 
66 /*
67  * globals
68  */
69 void *t_handle;			/* transport handle */
70 char aluaNode[256];		/* one of the two alua peers */
71 char myNode[256];		/* this hostname */
72 int log_debug = 0;
73 int fore_ground = 0;
74 int proxy_hdl;
75 pt_ops_t *pt_ops;
76 pthread_mutex_t send_mutex = PTHREAD_MUTEX_INITIALIZER;
77 
78 /*
79  * killHandler
80  *
81  * Terminates this process on SIGQUIT, SIGINT, SIGTERM
82  */
83 /* ARGSUSED */
84 static void
killHandler(int sig)85 killHandler(int sig)
86 {
87 	exit(0);
88 }
89 
90 /*
91  * doorHandler
92  *
93  * Recieve data from the local proxy port provider and relay
94  * it to the peer node.
95  */
96 /* ARGSUSED */
97 void
doorHandler(void * cookie,char * args,size_t alen,door_desc_t * ddp,uint_t ndid)98 doorHandler(
99 	void		*cookie,
100 	char		*args,
101 	size_t		alen,
102 	door_desc_t	*ddp,
103 	uint_t		ndid)
104 {
105 	uint32_t result = 0;
106 
107 	if (ddp != NULL || ndid != 0) {
108 		syslog(LOG_DAEMON|LOG_WARNING,
109 		    "descriptor passed to door %p %d", ddp, ndid);
110 		result = EINVAL;
111 	}
112 
113 	if (args == NULL || alen == 0) {
114 		syslog(LOG_DAEMON|LOG_WARNING,
115 		    "empty message passed to door %p %d", args, alen);
116 		result = EFAULT;
117 	}
118 
119 	if (result == 0)
120 		result = postMsg((uint_t)alen, (uchar_t *)args);
121 	(void) door_return((char *)&result, sizeof (result), NULL, 0);
122 
123 	syslog(LOG_DAEMON|LOG_WARNING, "door_return FAILED %d", errno);
124 	exit(errno);
125 }
126 
127 static int
postMsg(uint_t nelem,uchar_t * aluaMsg)128 postMsg(uint_t nelem, uchar_t *aluaMsg)
129 {
130 	uint32_t buflen;
131 	uchar_t *buf;
132 	int ret = 0;
133 	int ns;
134 
135 	if (t_handle == NULL) {
136 		syslog(LOG_DAEMON|LOG_WARNING,
137 		    "postMsg() no transport handle");
138 		exit(1);
139 	}
140 
141 	buf = malloc(nelem + sizeof (buflen));
142 
143 	buflen = htonl(nelem);	/* length in network byte order */
144 	bcopy(&buflen, buf, sizeof (buflen));
145 	bcopy(aluaMsg, buf + sizeof (buflen), nelem);
146 
147 	(void) pthread_mutex_lock(&send_mutex);
148 	ns = pt_ops->stmf_proxy_send(t_handle, buf, nelem + sizeof (buflen));
149 	(void) pthread_mutex_unlock(&send_mutex);
150 	if (ns != nelem + sizeof (buflen)) {
151 		ret = errno;
152 		if (ret == 0)
153 			ret = ENOTTY;	/* something bogus */
154 		syslog(LOG_DAEMON|LOG_CRIT, "send() call failed: %d", ret);
155 	}
156 	free(buf);
157 	return (ret);
158 }
159 
160 /*
161  * Multi-thread the data path from the peer node to the local
162  * proxy port provider. During discover, there can be a large
163  * burst of messages from the peer node proportional to the number
164  * of LUs. Multiple threads allow these messages to be processed
165  * simultaneously.
166  */
167 typedef struct pppt_drv_queue {
168 	struct pppt_drv_queue	*next;
169 	uint32_t		buflen;
170 	uchar_t			*buf;
171 } pppt_drv_queue_t;
172 
173 pppt_drv_queue_t *pq_head = NULL;
174 pthread_mutex_t pq_mutex = PTHREAD_MUTEX_INITIALIZER;
175 pthread_cond_t pq_cond = PTHREAD_COND_INITIALIZER;
176 int pq_num_threads = 0;
177 int pq_avail_threads = 0;
178 
179 /*ARGSUSED*/
180 void *
push_to_drv(void * arg)181 push_to_drv(void *arg)
182 {
183 	pppt_drv_queue_t	*pq;
184 	int rc;
185 
186 	(void) pthread_mutex_lock(&pq_mutex);
187 	pq_num_threads++;
188 	(void) pthread_mutex_unlock(&pq_mutex);
189 	for (;;) {
190 		(void) pthread_mutex_lock(&pq_mutex);
191 		while (pq_head == NULL) {
192 			pq_avail_threads++;
193 			(void) pthread_cond_wait(&pq_cond, &pq_mutex);
194 			pq_avail_threads--;
195 		}
196 		pq = pq_head;
197 		pq_head = pq->next;
198 		pq->next = NULL;
199 		(void) pthread_mutex_unlock(&pq_mutex);
200 		/* Relay the message to the local kernel */
201 		rc = stmfPostProxyMsg(proxy_hdl, (void *)pq->buf, pq->buflen);
202 		if (rc != STMF_STATUS_SUCCESS) {
203 			/* XXX die ? */
204 			syslog(LOG_DAEMON|LOG_CRIT, "ioctl failed - %d", errno);
205 		}
206 		free(pq->buf);
207 		free(pq);
208 	}
209 	/*NOTREACHED*/
210 	return (NULL);
211 }
212 
213 /*
214  * Receive data from peer and queue it up for the proxy driver.
215  */
216 int message_count = 0;
217 static void
relay_peer_msg()218 relay_peer_msg()
219 {
220 	uint32_t		buflen;
221 	pppt_drv_queue_t	*pq, *tmpq;
222 	pthread_t		tid;
223 	int			rc;
224 
225 
226 	/* first receive the length of the message */
227 	if ((pt_ops->stmf_proxy_recv(t_handle, (uchar_t *)&buflen,
228 	    sizeof (buflen))) != sizeof (buflen)) {
229 		syslog(LOG_DAEMON|LOG_WARNING, "recv() call failed: %d",
230 		    errno);
231 		exit(1);
232 	}
233 
234 	pq = malloc(sizeof (*pq));
235 	pq->next = NULL;
236 	pq->buflen = ntohl(buflen);
237 	pq->buf = malloc(pq->buflen+4);
238 	if (log_debug) {
239 		syslog(LOG_DAEMON|LOG_DEBUG,
240 		    "recvMsg: size of buffer - %d", (int)pq->buflen);
241 	}
242 
243 	if ((pt_ops->stmf_proxy_recv(t_handle, pq->buf, pq->buflen)) !=
244 	    pq->buflen) {
245 		syslog(LOG_DAEMON|LOG_WARNING, "recv() call failed: %d",
246 		    errno);
247 		exit(1);
248 	}
249 
250 	/* Eat the first message from peer */
251 	if (message_count++ == 0) {
252 		*(pq->buf+pq->buflen) = 0;
253 		free(pq->buf);
254 		free(pq);
255 		return;
256 	}
257 
258 	/* Queue the message to the driver */
259 	(void) pthread_mutex_lock(&pq_mutex);
260 	if (pq_head == NULL) {
261 		pq_head = pq;
262 	} else {
263 		/* add to the tail */
264 		tmpq = pq_head;
265 		while (tmpq->next != NULL)
266 			tmpq = tmpq->next;
267 		tmpq->next = pq;
268 	}
269 
270 	/* Make sure there is a thread to service this message */
271 	if (pq_avail_threads) {
272 		/* wake an available thread */
273 		(void) pthread_cond_signal(&pq_cond);
274 		(void) pthread_mutex_unlock(&pq_mutex);
275 	} else {
276 		/* no threads available, create a new thread */
277 		(void) pthread_mutex_unlock(&pq_mutex);
278 		rc = pthread_create(&tid, NULL, push_to_drv, NULL);
279 		if (rc != 0) {
280 			syslog(LOG_DAEMON|LOG_WARNING,
281 			    "pthread_create() call failed: %d", rc);
282 			if (pq_num_threads == 0) {
283 				/* never created a thread */
284 				exit(rc);
285 			}
286 		}
287 	}
288 }
289 
290 /*
291  * Initialization for a daemon process
292  */
293 static void
daemonInit(void)294 daemonInit(void)
295 {
296 	pid_t	pid;
297 	int	devnull;
298 
299 	if (fore_ground)
300 		return;
301 
302 	if ((pid = fork()) < 0) {
303 		syslog(LOG_DAEMON|LOG_CRIT, "Could not fork(). Exiting");
304 		exit(1);
305 	} else if (pid != 0) {
306 		/*
307 		 * XXX
308 		 * Simple approach for now - let the service go online.
309 		 * Later, set-up a pipe to the child and wait until the
310 		 * child indicates service is setup.
311 		 */
312 		exit(SMF_EXIT_OK);
313 	}
314 
315 	(void) setsid();
316 
317 	(void) chdir("/");
318 
319 	(void) umask(0);
320 
321 
322 	devnull = open("/dev/null", O_RDWR);
323 	if (devnull < 0) {
324 		syslog(LOG_DAEMON|LOG_CRIT,
325 		    "Failed to open /dev/null. Exiting");
326 		exit(1);
327 	}
328 
329 	(void) dup2(devnull, STDIN_FILENO);
330 	(void) dup2(devnull, STDOUT_FILENO);
331 	(void) dup2(devnull, STDERR_FILENO);
332 	(void) close(devnull);
333 }
334 
335 void
daemon_fini(int rc)336 daemon_fini(int rc)
337 {
338 	/*
339 	 * XXX inform the parent about the service state
340 	 * For now, just exit on error.
341 	 */
342 	if (rc != 0)
343 		exit(rc);
344 }
345 
346 static int
open_proxy_driver()347 open_proxy_driver()
348 {
349 	int drv_door_fd;
350 	int stmf_ret;
351 
352 	/*
353 	 * Create communication channel for the driver.
354 	 */
355 	if ((drv_door_fd = door_create(doorHandler, NULL, 0)) < 0) {
356 		perror("door_create");
357 		syslog(LOG_DAEMON|LOG_DEBUG,
358 		    "could not create door: errno %d", errno);
359 		return (SMF_EXIT_ERR_FATAL);
360 	}
361 
362 	stmf_ret = stmfInitProxyDoor(&proxy_hdl, drv_door_fd);
363 	if (stmf_ret != STMF_STATUS_SUCCESS) {
364 		perror("pppt ioctl: door install");
365 		syslog(LOG_DAEMON|LOG_DEBUG,
366 		    "could not install door: errno %d", errno);
367 		return (SMF_EXIT_ERR_FATAL);
368 	}
369 
370 	return (SMF_EXIT_OK);
371 }
372 
373 /*
374  * daemon entry
375  *
376  * parse arguments
377  * create resources to talk to child
378  * if !foreground
379  *    daemonize, run as child
380  * open proxy driver
381  * install door in proxy driver
382  * create socket
383  * if server-side
384  *     bind socket
385  * if !foreground
386  *    inform parent things aok
387  * if parent
388  *    exit(SMF_EXIT_OK)
389  * if server-side
390  *    accept
391  * if client-side
392  *    connect
393  * send hello
394  * recv hello
395  * loop on recieve
396  * XXX anyway to check in envp that we are started by SMF?
397  */
398 int
main(int argc,char * argv[])399 main(int argc, char *argv[])
400 {
401 	struct	sockaddr_in sin;
402 	int	rc;
403 	struct sigaction 	act;
404 	sigset_t		sigmask;
405 	int	c;
406 	int	node = 0;
407 	int	node_override = 0;
408 	int	server_node = 0;
409 	int	server_match = 0;
410 	extern char *optarg;
411 	int stmf_ret;
412 
413 	(void) setlocale(LC_ALL, "");
414 	openlog("stmfproxy", LOG_PID, LOG_DAEMON);
415 	(void) setlogmask(LOG_UPTO(LOG_INFO));
416 
417 	while ((c = getopt(argc, argv, "dfn:")) != -1) {
418 		switch (c) {
419 			case 'd':
420 				(void) setlogmask(LOG_UPTO(LOG_DEBUG));
421 				log_debug = 1;
422 				break;
423 			case 'f':
424 				fore_ground = 1;
425 				break;
426 			case 'n':
427 				node_override = 1;
428 				node = atoi(optarg);
429 				break;
430 			default:
431 				/*
432 				 * Should never happen from smf
433 				 */
434 				(void) fprintf(stderr, USAGE, argv[0]);
435 				exit(SMF_EXIT_ERR_CONFIG);
436 				break;
437 		}
438 	}
439 	/*
440 	 * After the options, only the server argument should remain.
441 	 */
442 	if (optind != argc-1) {
443 		(void) fprintf(stderr, USAGE, argv[0]);
444 		exit(SMF_EXIT_ERR_CONFIG);
445 	}
446 	(void) strcpy(aluaNode, argv[optind]);
447 	syslog(LOG_DAEMON|LOG_DEBUG, "aluaNode %s", aluaNode);
448 	if (gethostname(myNode, 255)) {
449 		perror("gethostname");
450 		exit(1);
451 	}
452 	if ((inet_aton(aluaNode, &sin.sin_addr)) == 0) {
453 		/*
454 		 * Not ipaddr, try hostname match.
455 		 */
456 		server_match = (strcmp(aluaNode, myNode)) ? 0 : 1;
457 	} else {
458 		/*
459 		 * see if this is our ip address
460 		 */
461 		(void) fprintf(stderr, "Sorry, cannot use ip adress format\n");
462 	}
463 	if (server_match) {
464 		server_node = 1;
465 		if (!node_override)
466 			node = 1;
467 	}
468 
469 
470 	/*
471 	 * Allow SIGQUIT, SIGINT and SIGTERM signals to terminate us
472 	 */
473 	act.sa_handler = killHandler;
474 	(void) sigemptyset(&act.sa_mask);
475 	act.sa_flags = 0;
476 
477 	/* Install the signal handler */
478 	(void) sigaction(SIGQUIT, &act, NULL);
479 	(void) sigaction(SIGINT, &act, NULL);
480 	(void) sigaction(SIGTERM, &act, NULL);
481 	(void) sigaction(SIGHUP, &act, NULL);
482 
483 	/* block all signals */
484 	(void) sigfillset(&sigmask);
485 
486 	/* unblock SIGQUIT, SIGINT, SIGTERM */
487 	(void) sigdelset(&sigmask, SIGQUIT);
488 	(void) sigdelset(&sigmask, SIGINT);
489 	(void) sigdelset(&sigmask, SIGTERM);
490 	(void) sigdelset(&sigmask, SIGHUP);
491 
492 	(void) sigprocmask(SIG_SETMASK, &sigmask, NULL);
493 
494 	/* time to go backstage */
495 	daemonInit();
496 
497 	if ((rc = open_proxy_driver()) != 0)
498 		daemon_fini(rc);
499 
500 	if ((rc = stmf_proxy_transport_init("sockets", &pt_ops)) != 0)
501 		daemon_fini(rc);
502 
503 	/*
504 	 * Establish connection
505 	 *
506 	 * At this point, the parent has exited and the service
507 	 * is online. But there are no real proxy services until
508 	 * this connect call succeeds. That could take a long time if
509 	 * the peer node is down.
510 	 */
511 	t_handle = pt_ops->stmf_proxy_connect(server_node, aluaNode);
512 	if (t_handle == NULL) {
513 		syslog(LOG_DAEMON|LOG_WARNING,
514 		    "socket() call failed: %d", errno);
515 		exit(1);
516 	}
517 
518 	/* The first message is a greeting */
519 	(void) postMsg((uint_t)strlen(myNode)+1, (uchar_t *)myNode);
520 	/* Read the greeting from peer node */
521 	relay_peer_msg();
522 	/*
523 	 * Set the alua state in stmf. No need to keep
524 	 * the device open since the proxy driver has a reference.
525 	 */
526 	stmf_ret = stmfSetAluaState(B_TRUE, node);
527 	if (stmf_ret != STMF_STATUS_SUCCESS) {
528 		syslog(LOG_DAEMON|LOG_CRIT, "stmf ioctl failed - %x", stmf_ret);
529 		exit(1);
530 	}
531 
532 	/* service is online */
533 	daemon_fini(0);
534 
535 	/*
536 	 * Loop relaying data from the peer daemon to the local kernel.
537 	 * Data coming from the local kernel is handled asynchronously
538 	 * by the door server.
539 	 */
540 	for (;;) { /* loop forever */
541 		relay_peer_msg();
542 	}
543 }
544