1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2017 Toomas Soome <tsoome@me.com>
14  * Copyright 2019, Joyent, Inc.
15  */
16 
17 /*
18  * This module adds support for loading and booting illumos multiboot2
19  * kernel. This code is only built to support the illumos kernel, it does
20  * not support xen.
21  */
22 
23 #include <sys/cdefs.h>
24 #include <sys/stddef.h>
25 
26 #include <sys/param.h>
27 #include <sys/exec.h>
28 #include <sys/linker.h>
29 #include <sys/module.h>
30 #include <sys/stdint.h>
31 #include <sys/multiboot2.h>
32 #include <stand.h>
33 #include <stdbool.h>
34 #include <machine/elf.h>
35 #include "libzfs.h"
36 
37 #include "bootstrap.h"
38 #include <sys/consplat.h>
39 
40 #include <machine/metadata.h>
41 #include <machine/pc/bios.h>
42 
43 #define	SUPPORT_DHCP
44 #include <bootp.h>
45 
46 #if !defined(EFI)
47 #include "../i386/btx/lib/btxv86.h"
48 #include "libi386.h"
49 #include "vbe.h"
50 
51 #else
52 #include <efi.h>
53 #include <efilib.h>
54 #include "loader_efi.h"
55 
56 static void (*trampoline)(uint32_t, struct relocator *, uint64_t);
57 static UINTN efi_map_size;		/* size of efi memory map */
58 #endif
59 
60 #include "platform/acfreebsd.h"
61 #include "acconfig.h"
62 #define	ACPI_SYSTEM_XFACE
63 #include "actypes.h"
64 #include "actbl.h"
65 
66 extern ACPI_TABLE_RSDP *rsdp;
67 
68 /* MB data heap pointer. */
69 static vm_offset_t last_addr;
70 
71 static int multiboot2_loadfile(char *, uint64_t, struct preloaded_file **);
72 static int multiboot2_exec(struct preloaded_file *);
73 
74 struct file_format multiboot2 = { multiboot2_loadfile, multiboot2_exec };
75 static bool keep_bs = false;
76 static bool have_framebuffer = false;
77 static vm_offset_t load_addr;
78 static vm_offset_t entry_addr;
79 bool has_boot_services = true;
80 
81 /*
82  * Validate tags in info request. This function is provided just to
83  * recognize the current tag list and only serves as a limited
84  * safe guard against possibly corrupt information.
85  */
86 static bool
is_info_request_valid(multiboot_header_tag_information_request_t * rtag)87 is_info_request_valid(multiboot_header_tag_information_request_t *rtag)
88 {
89 	int i;
90 
91 	/*
92 	 * If the tag is optional and we do not support it, we do not
93 	 * have to do anything special, so we skip optional tags.
94 	 */
95 	if (rtag->mbh_flags & MULTIBOOT_HEADER_TAG_OPTIONAL)
96 		return (true);
97 
98 	for (i = 0; i < (rtag->mbh_size - sizeof (*rtag)) /
99 	    sizeof (rtag->mbh_requests[0]); i++)
100 		switch (rtag->mbh_requests[i]) {
101 		case MULTIBOOT_TAG_TYPE_END:
102 		case MULTIBOOT_TAG_TYPE_CMDLINE:
103 		case MULTIBOOT_TAG_TYPE_BOOT_LOADER_NAME:
104 		case MULTIBOOT_TAG_TYPE_MODULE:
105 		case MULTIBOOT_TAG_TYPE_BASIC_MEMINFO:
106 		case MULTIBOOT_TAG_TYPE_BOOTDEV:
107 		case MULTIBOOT_TAG_TYPE_MMAP:
108 		case MULTIBOOT_TAG_TYPE_FRAMEBUFFER:
109 		case MULTIBOOT_TAG_TYPE_VBE:
110 		case MULTIBOOT_TAG_TYPE_ELF_SECTIONS:
111 		case MULTIBOOT_TAG_TYPE_APM:
112 		case MULTIBOOT_TAG_TYPE_EFI32:
113 		case MULTIBOOT_TAG_TYPE_EFI64:
114 		case MULTIBOOT_TAG_TYPE_ACPI_OLD:
115 		case MULTIBOOT_TAG_TYPE_ACPI_NEW:
116 		case MULTIBOOT_TAG_TYPE_NETWORK:
117 		case MULTIBOOT_TAG_TYPE_EFI_MMAP:
118 		case MULTIBOOT_TAG_TYPE_EFI_BS:
119 		case MULTIBOOT_TAG_TYPE_EFI32_IH:
120 		case MULTIBOOT_TAG_TYPE_EFI64_IH:
121 		case MULTIBOOT_TAG_TYPE_LOAD_BASE_ADDR:
122 			break;
123 		default:
124 			printf("unsupported information tag: 0x%x\n",
125 			    rtag->mbh_requests[i]);
126 			return (false);
127 		}
128 	return (true);
129 }
130 
131 static int
multiboot2_loadfile(char * filename,uint64_t dest,struct preloaded_file ** result)132 multiboot2_loadfile(char *filename, uint64_t dest,
133     struct preloaded_file **result)
134 {
135 	int fd, error;
136 	uint32_t i;
137 	struct stat st;
138 	caddr_t header_search;
139 	multiboot2_header_t *header;
140 	multiboot_header_tag_t *tag;
141 	multiboot_header_tag_address_t *addr_tag = NULL;
142 	multiboot_header_tag_entry_address_t *entry_tag = NULL;
143 	struct preloaded_file *fp;
144 
145 	/* This allows to check other file formats from file_formats array. */
146 	error = EFTYPE;
147 	if (filename == NULL)
148 		return (error);
149 
150 	/* is kernel already loaded? */
151 	fp = file_findfile(NULL, NULL);
152 	if (fp != NULL)
153 		return (error);
154 
155 	if ((fd = open(filename, O_RDONLY)) == -1)
156 		return (errno);
157 
158 	/*
159 	 * Read MULTIBOOT_SEARCH size in order to search for the
160 	 * multiboot magic header.
161 	 */
162 	header_search = malloc(MULTIBOOT_SEARCH);
163 	if (header_search == NULL) {
164 		close(fd);
165 		return (ENOMEM);
166 	}
167 
168 	if (read(fd, header_search, MULTIBOOT_SEARCH) != MULTIBOOT_SEARCH)
169 		goto out;
170 
171 	header = NULL;
172 	for (i = 0; i <= (MULTIBOOT_SEARCH - sizeof (multiboot2_header_t));
173 	    i += MULTIBOOT_HEADER_ALIGN) {
174 		header = (multiboot2_header_t *)(header_search + i);
175 
176 		/* Do we have match on magic? */
177 		if (header->mb2_magic != MULTIBOOT2_HEADER_MAGIC) {
178 			header = NULL;
179 			continue;
180 		}
181 		/*
182 		 * Validate checksum, the sum of magic + architecture +
183 		 * header_length + checksum must equal 0.
184 		 */
185 		if (header->mb2_magic + header->mb2_architecture +
186 		    header->mb2_header_length + header->mb2_checksum != 0) {
187 			header = NULL;
188 			continue;
189 		}
190 		/*
191 		 * Finally, the entire header must fit within MULTIBOOT_SEARCH.
192 		 */
193 		if (i + header->mb2_header_length > MULTIBOOT_SEARCH) {
194 			header = NULL;
195 			continue;
196 		}
197 		break;
198 	}
199 
200 	if (header == NULL)
201 		goto out;
202 
203 	have_framebuffer = false;
204 	for (tag = header->mb2_tags; tag->mbh_type != MULTIBOOT_TAG_TYPE_END;
205 	    tag = (multiboot_header_tag_t *)((uintptr_t)tag +
206 	    roundup2(tag->mbh_size, MULTIBOOT_TAG_ALIGN))) {
207 		switch (tag->mbh_type) {
208 		case MULTIBOOT_HEADER_TAG_INFORMATION_REQUEST:
209 			if (is_info_request_valid((void*)tag) == false)
210 				goto out;
211 			break;
212 		case MULTIBOOT_HEADER_TAG_ADDRESS:
213 			addr_tag = (multiboot_header_tag_address_t *)tag;
214 			break;
215 		case MULTIBOOT_HEADER_TAG_ENTRY_ADDRESS:
216 			entry_tag =
217 			    (multiboot_header_tag_entry_address_t *)tag;
218 			break;
219 		case MULTIBOOT_HEADER_TAG_CONSOLE_FLAGS:
220 			break;
221 		case MULTIBOOT_HEADER_TAG_FRAMEBUFFER:
222 			have_framebuffer = true;
223 			break;
224 		case MULTIBOOT_HEADER_TAG_MODULE_ALIGN:
225 			/* we always align modules */
226 			break;
227 		case MULTIBOOT_HEADER_TAG_EFI_BS:
228 			keep_bs = true;
229 			break;
230 		default:
231 			if (!(tag->mbh_flags & MULTIBOOT_HEADER_TAG_OPTIONAL)) {
232 				printf("unsupported tag: 0x%x\n",
233 				    tag->mbh_type);
234 				goto out;
235 			}
236 		}
237 	}
238 
239 	/*
240 	 * We must have addr_tag and entry_tag to load a 64-bit kernel.
241 	 * If these tags are missing, we either have a 32-bit kernel, or
242 	 * this is not our kernel at all.
243 	 */
244 	if (addr_tag != NULL && entry_tag != NULL) {
245 		fp = file_alloc();
246 		if (fp == NULL) {
247 			error = ENOMEM;
248 			goto out;
249 		}
250 		if (lseek(fd, 0, SEEK_SET) == -1) {
251 			printf("lseek failed\n");
252 			error = EIO;
253 			file_discard(fp);
254 			goto out;
255 		}
256 		if (fstat(fd, &st) < 0) {
257 			printf("fstat failed\n");
258 			error = EIO;
259 			file_discard(fp);
260 			goto out;
261 		}
262 
263 		load_addr = addr_tag->mbh_load_addr;
264 		entry_addr = entry_tag->mbh_entry_addr;
265 		fp->f_addr = archsw.arch_loadaddr(LOAD_KERN, filename,
266 		    addr_tag->mbh_load_addr);
267 		if (fp->f_addr == 0) {
268 			error = ENOMEM;
269 			file_discard(fp);
270 			goto out;
271 		}
272 		fp->f_size = archsw.arch_readin(fd, fp->f_addr, st.st_size);
273 
274 		if (fp->f_size != st.st_size) {
275 			printf("error reading %s: %s\n", filename,
276 			    strerror(errno));
277 			file_discard(fp);
278 			error = EIO;
279 			goto out;
280 		}
281 
282 		fp->f_name = strdup(filename);
283 		fp->f_type = strdup("aout multiboot2 kernel");
284 		if (fp->f_name == NULL || fp->f_type == NULL) {
285 			error = ENOMEM;
286 			file_discard(fp);
287 			goto out;
288 		}
289 
290 		fp->f_metadata = NULL;
291 		error = 0;
292 	} else {
293 #if defined(EFI)
294 		/* 32-bit kernel is not yet supported for EFI */
295 		printf("32-bit kernel is not supported by UEFI loader\n");
296 		error = ENOTSUP;
297 		goto out;
298 #endif
299 		/* elf32_loadfile_raw will fill the attributes in fp. */
300 		error = elf32_loadfile_raw(filename, dest, &fp, 2);
301 		if (error != 0) {
302 			printf("elf32_loadfile_raw failed: %d unable to "
303 			    "load multiboot2 kernel\n", error);
304 			goto out;
305 		}
306 		entry_addr = fp->f_addr;
307 		/*
308 		 * We want the load_addr to have some legal value,
309 		 * so we set it same as the entry_addr.
310 		 * The distinction is important with UEFI, but not
311 		 * with BIOS version, because BIOS version does not use
312 		 * staging area.
313 		 */
314 		load_addr = fp->f_addr;
315 	}
316 
317 	setenv("kernelname", fp->f_name, 1);
318 #if defined(EFI)
319 	efi_addsmapdata(fp);
320 #else
321 	bios_addsmapdata(fp);
322 #endif
323 	*result = fp;
324 out:
325 	free(header_search);
326 	close(fd);
327 	return (error);
328 }
329 
330 /*
331  * Search the command line for named property.
332  *
333  * Return codes:
334  *	0	The name is found, we return the data in value and len.
335  *	ENOENT	The name is not found.
336  *	EINVAL	The provided command line is badly formed.
337  */
338 static int
find_property_value(const char * cmd,const char * name,const char ** value,size_t * len)339 find_property_value(const char *cmd, const char *name, const char **value,
340     size_t *len)
341 {
342 	const char *namep, *valuep;
343 	size_t name_len, value_len;
344 	int quoted;
345 
346 	*value = NULL;
347 	*len = 0;
348 
349 	if (cmd == NULL)
350 		return (ENOENT);
351 
352 	while (*cmd != '\0') {
353 		if (cmd[0] != '-' || cmd[1] != 'B') {
354 			cmd++;
355 			continue;
356 		}
357 		cmd += 2;	/* Skip -B */
358 		while (cmd[0] == ' ' || cmd[0] == '\t')
359 			cmd++;	/* Skip whitespaces. */
360 		while (*cmd != '\0' && cmd[0] != ' ' && cmd[0] != '\t') {
361 			namep = cmd;
362 			valuep = strchr(cmd, '=');
363 			if (valuep == NULL)
364 				break;
365 			name_len = valuep - namep;
366 			valuep++;
367 			value_len = 0;
368 			quoted = 0;
369 			for (; ; ++value_len) {
370 				if (valuep[value_len] == '\0')
371 					break;
372 
373 				/* Is this value quoted? */
374 				if (value_len == 0 &&
375 				    (valuep[0] == '\'' || valuep[0] == '"')) {
376 					quoted = valuep[0];
377 					++value_len;
378 				}
379 
380 				/*
381 				 * In the quote accept any character,
382 				 * but look for ending quote.
383 				 */
384 				if (quoted != 0) {
385 					if (valuep[value_len] == quoted)
386 						quoted = 0;
387 					continue;
388 				}
389 
390 				/* A comma or white space ends the value. */
391 				if (valuep[value_len] == ',' ||
392 				    valuep[value_len] == ' ' ||
393 				    valuep[value_len] == '\t')
394 					break;
395 			}
396 			if (quoted != 0) {
397 				printf("Missing closing '%c' in \"%s\"\n",
398 				    quoted, valuep);
399 				return (EINVAL);
400 			}
401 			if (value_len != 0) {
402 				if (strncmp(namep, name, name_len) == 0) {
403 					*value = valuep;
404 					*len = value_len;
405 					return (0);
406 				}
407 			}
408 			cmd = valuep + value_len;
409 			while (*cmd == ',')
410 				cmd++;
411 		}
412 	}
413 	return (ENOENT);
414 }
415 
416 /*
417  * If command line has " -B ", insert property after "-B ", otherwise
418  * append to command line.
419  */
420 static char *
insert_cmdline(const char * head,const char * prop)421 insert_cmdline(const char *head, const char *prop)
422 {
423 	const char *prop_opt = " -B ";
424 	char *cmdline, *tail;
425 	int len = 0;
426 
427 	tail = strstr(head, prop_opt);
428 	if (tail != NULL) {
429 		ptrdiff_t diff;
430 		tail += strlen(prop_opt);
431 		diff = tail - head;
432 		if (diff >= INT_MAX)
433 			return (NULL);
434 		len = (int)diff;
435 	}
436 
437 	if (tail == NULL)
438 		asprintf(&cmdline, "%s%s%s", head, prop_opt, prop);
439 	else
440 		asprintf(&cmdline, "%.*s%s,%s", len, head, prop, tail);
441 
442 	return (cmdline);
443 }
444 
445 /*
446  * Since we have no way to pass the environment to the mb1 kernel other than
447  * through arguments, we need to take care of console setup.
448  *
449  * If the console is in mirror mode, set the kernel console from $os_console.
450  * If it's unset, use first item from $console.
451  * If $console is "ttyX", also pass $ttyX-mode, since it may have been set by
452  * the user.
453  *
454  * In case of memory allocation errors, just return the original command line
455  * so we have a chance of booting.
456  *
457  * On success, cl will be freed and a new, allocated command line string is
458  * returned.
459  *
460  * For the mb2 kernel, we only set command line console if os_console is set.
461  * We can not overwrite console in the environment, as it can disrupt the
462  * loader console messages, and we do not want to deal with the os_console
463  * in the kernel.
464  */
465 static char *
update_cmdline(char * cl,bool mb2)466 update_cmdline(char *cl, bool mb2)
467 {
468 	char *os_console = getenv("os_console");
469 	char *ttymode = NULL;
470 	char mode[10];
471 	char *tmp;
472 	const char *prop;
473 	size_t plen;
474 	int rv;
475 
476 	if (mb2 == true && os_console == NULL)
477 		return (cl);
478 
479 	if (os_console == NULL) {
480 		tmp = strdup(getenv("console"));
481 		os_console = strsep(&tmp, ", ");
482 	} else {
483 		os_console = strdup(os_console);
484 	}
485 
486 	if (os_console == NULL)
487 		return (cl);
488 
489 	if (mb2 == false && strncmp(os_console, "tty", 3) == 0) {
490 		snprintf(mode, sizeof (mode), "%s-mode", os_console);
491 		/*
492 		 * The ttyX-mode variable is set by our serial console
493 		 * driver for ttya-ttyd. However, since the os_console
494 		 * values are not verified, it is possible we get bogus
495 		 * name and no mode variable. If so, we do not set console
496 		 * property and let the kernel use defaults.
497 		 */
498 		if ((ttymode = getenv(mode)) == NULL)
499 			return (cl);
500 	}
501 
502 	rv = find_property_value(cl, "console", &prop, &plen);
503 	if (rv != 0 && rv != ENOENT) {
504 		free(os_console);
505 		return (cl);
506 	}
507 
508 	/* If console is set and this is MB2 boot, we are done. */
509 	if (rv == 0 && mb2 == true) {
510 		free(os_console);
511 		return (cl);
512 	}
513 
514 	/* If console is set, do we need to set tty mode? */
515 	if (rv == 0) {
516 		const char *ttyp = NULL;
517 		size_t ttylen;
518 
519 		free(os_console);
520 		os_console = NULL;
521 		*mode = '\0';
522 		if (strncmp(prop, "tty", 3) == 0 && plen == 4) {
523 			strncpy(mode, prop, plen);
524 			mode[plen] = '\0';
525 			strncat(mode, "-mode", 5);
526 			find_property_value(cl, mode, &ttyp, &ttylen);
527 		}
528 
529 		if (*mode != '\0' && ttyp == NULL)
530 			ttymode = getenv(mode);
531 		else
532 			return (cl);
533 	}
534 
535 	/* Build updated command line. */
536 	if (os_console != NULL) {
537 		char *propstr;
538 
539 		asprintf(&propstr, "console=%s", os_console);
540 		free(os_console);
541 		if (propstr == NULL) {
542 			return (cl);
543 		}
544 
545 		tmp = insert_cmdline(cl, propstr);
546 		free(propstr);
547 		if (tmp == NULL)
548 			return (cl);
549 
550 		free(cl);
551 		cl = tmp;
552 	}
553 	if (ttymode != NULL) {
554 		char *propstr;
555 
556 		asprintf(&propstr, "%s=\"%s\"", mode, ttymode);
557 		if (propstr == NULL)
558 			return (cl);
559 
560 		tmp = insert_cmdline(cl, propstr);
561 		free(propstr);
562 		if (tmp == NULL)
563 			return (cl);
564 		free(cl);
565 		cl = tmp;
566 	}
567 
568 	return (cl);
569 }
570 
571 /*
572  * Build the kernel command line. Shared function between MB1 and MB2.
573  *
574  * In both cases, if fstype is set and is not zfs, we do not set up
575  * zfs-bootfs property. But we set kernel file name and options.
576  *
577  * For the MB1, we only can pass properties on command line, so
578  * we will set console, ttyX-mode (for serial console) and zfs-bootfs.
579  *
580  * For the MB2, we can pass properties in environment, but if os_console
581  * is set in environment, we need to add console property on the kernel
582  * command line.
583  *
584  * The console properties are managed in update_cmdline().
585  */
586 int
mb_kernel_cmdline(struct preloaded_file * fp,struct devdesc * rootdev,char ** line)587 mb_kernel_cmdline(struct preloaded_file *fp, struct devdesc *rootdev,
588     char **line)
589 {
590 	const char *fs = getenv("fstype");
591 	char *cmdline;
592 	size_t len;
593 	bool zfs_root = false;
594 	bool mb2;
595 	int rv;
596 
597 	/*
598 	 * 64-bit kernel has aout header, 32-bit kernel is elf, and the
599 	 * type strings are different. Lets just search for "multiboot2".
600 	 */
601 	if (strstr(fp->f_type, "multiboot2") == NULL)
602 		mb2 = false;
603 	else
604 		mb2 = true;
605 
606 	if (rootdev->d_dev->dv_type == DEVT_ZFS)
607 		zfs_root = true;
608 
609 	/* If we have fstype set in env, reset zfs_root if needed. */
610 	if (fs != NULL && strcmp(fs, "zfs") != 0)
611 		zfs_root = false;
612 
613 	/*
614 	 * If we have fstype set on the command line,
615 	 * reset zfs_root if needed.
616 	 */
617 	rv = find_property_value(fp->f_args, "fstype", &fs, &len);
618 	if (rv != 0 && rv != ENOENT)
619 		return (rv);
620 
621 	if (fs != NULL && strncmp(fs, "zfs", len) != 0)
622 		zfs_root = false;
623 
624 	/* zfs_bootfs() will set the environment, it must be called. */
625 	if (zfs_root == true)
626 		fs = zfs_bootfs(rootdev);
627 
628 	if (fp->f_args == NULL)
629 		cmdline = strdup(fp->f_name);
630 	else
631 		asprintf(&cmdline, "%s %s", fp->f_name, fp->f_args);
632 
633 	if (cmdline == NULL)
634 		return (ENOMEM);
635 
636 	/* Append zfs-bootfs for MB1 command line. */
637 	if (mb2 == false && zfs_root == true) {
638 		char *tmp;
639 
640 		tmp = insert_cmdline(cmdline, fs);
641 		free(cmdline);
642 		if (tmp == NULL)
643 			return (ENOMEM);
644 		cmdline = tmp;
645 	}
646 
647 	*line = update_cmdline(cmdline, mb2);
648 	return (0);
649 }
650 
651 /*
652  * Returns allocated virtual address from MB info area.
653  */
654 static vm_offset_t
mb_malloc(size_t n)655 mb_malloc(size_t n)
656 {
657 	vm_offset_t ptr = last_addr;
658 	last_addr = roundup(last_addr + n, MULTIBOOT_TAG_ALIGN);
659 	return (ptr);
660 }
661 
662 /*
663  * Calculate size for module tag list.
664  */
665 static size_t
module_size(struct preloaded_file * fp)666 module_size(struct preloaded_file *fp)
667 {
668 	size_t len, size;
669 	struct preloaded_file *mfp;
670 
671 	size = 0;
672 	for (mfp = fp->f_next; mfp != NULL; mfp = mfp->f_next) {
673 		len = strlen(mfp->f_name) + 1;
674 		len += strlen(mfp->f_type) + 5 + 1; /* 5 is for "type=" */
675 		if (mfp->f_args != NULL)
676 			len += strlen(mfp->f_args) + 1;
677 		size += sizeof (multiboot_tag_module_t) + len;
678 		size = roundup(size, MULTIBOOT_TAG_ALIGN);
679 	}
680 	return (size);
681 }
682 
683 #if defined(EFI)
684 /*
685  * Calculate size for UEFI memory map tag.
686  */
687 #define	EFI_EXTRA_PAGES	3
688 
689 static int
efimemmap_size(void)690 efimemmap_size(void)
691 {
692 	UINTN size, cur_size, desc_size;
693 	EFI_MEMORY_DESCRIPTOR *mmap;
694 	EFI_STATUS ret;
695 
696 	size = EFI_PAGE_SIZE;		/* Start with 4k. */
697 	while (1) {
698 		cur_size = size;
699 		mmap = malloc(cur_size);
700 		if (mmap == NULL)
701 			return (0);
702 		ret = BS->GetMemoryMap(&cur_size, mmap, NULL, &desc_size, NULL);
703 		free(mmap);
704 		if (ret == EFI_SUCCESS)
705 			break;
706 		if (ret == EFI_BUFFER_TOO_SMALL) {
707 			if (size < cur_size)
708 				size = cur_size;
709 			size += (EFI_PAGE_SIZE);
710 		} else
711 			return (0);
712 	}
713 
714 	/* EFI MMAP will grow when we allocate MBI, set some buffer. */
715 	size += (EFI_EXTRA_PAGES << EFI_PAGE_SHIFT);
716 	size = roundup2(size, EFI_PAGE_SIZE);
717 	efi_map_size = size;	/* Record the calculated size. */
718 	return (sizeof (multiboot_tag_efi_mmap_t) + size);
719 }
720 #endif
721 
722 /*
723  * Calculate size for bios smap tag.
724  */
725 static size_t
biossmap_size(struct preloaded_file * fp)726 biossmap_size(struct preloaded_file *fp)
727 {
728 	int num;
729 	struct file_metadata *md;
730 
731 	md = file_findmetadata(fp, MODINFOMD_SMAP);
732 	if (md == NULL)
733 		return (0);
734 
735 	num = md->md_size / sizeof (struct bios_smap); /* number of entries */
736 	return (sizeof (multiboot_tag_mmap_t) +
737 	    num * sizeof (multiboot_mmap_entry_t));
738 }
739 
740 static size_t
mbi_size(struct preloaded_file * fp,char * cmdline)741 mbi_size(struct preloaded_file *fp, char *cmdline)
742 {
743 	size_t size;
744 #if !defined(EFI)
745 	extern multiboot_tag_framebuffer_t gfx_fb;
746 #endif
747 
748 	size = sizeof (uint32_t) * 2; /* first 2 fields from MBI header */
749 	size += sizeof (multiboot_tag_string_t) + strlen(cmdline) + 1;
750 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
751 	size += sizeof (multiboot_tag_string_t) + strlen(bootprog_info) + 1;
752 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
753 #if !defined(EFI)
754 	size += sizeof (multiboot_tag_basic_meminfo_t);
755 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
756 #endif
757 	size += module_size(fp);
758 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
759 #if defined(EFI)
760 	size += sizeof (multiboot_tag_efi64_t);
761 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
762 	size += efimemmap_size();
763 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
764 
765 	if (have_framebuffer == true) {
766 		size += sizeof (multiboot_tag_framebuffer_t);
767 		size = roundup2(size, MULTIBOOT_TAG_ALIGN);
768 	}
769 #endif
770 
771 	size += biossmap_size(fp);
772 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
773 
774 #if !defined(EFI)
775 	if (gfx_fb.framebuffer_common.framebuffer_type ==
776 	    MULTIBOOT_FRAMEBUFFER_TYPE_INDEXED) {
777 		size += sizeof (struct multiboot_tag_framebuffer_common);
778 		size += CMAP_SIZE * sizeof (multiboot_color_t);
779 	} else {
780 		size += sizeof (multiboot_tag_framebuffer_t);
781 	}
782 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
783 
784 	size += sizeof (multiboot_tag_vbe_t);
785 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
786 #endif
787 
788 	if (bootp_response != NULL) {
789 		size += sizeof (multiboot_tag_network_t) + bootp_response_size;
790 		size = roundup2(size, MULTIBOOT_TAG_ALIGN);
791 	}
792 
793 	if (rsdp != NULL) {
794 		if (rsdp->Revision == 0) {
795 			size += sizeof (multiboot_tag_old_acpi_t) +
796 			    sizeof (ACPI_RSDP_COMMON);
797 		} else {
798 			size += sizeof (multiboot_tag_new_acpi_t) +
799 			    rsdp->Length;
800 		}
801 		size = roundup2(size, MULTIBOOT_TAG_ALIGN);
802 	}
803 	size += sizeof (multiboot_tag_t);
804 
805 	return (size);
806 }
807 
808 #if defined(EFI)
809 static bool
overlaps(uintptr_t start1,size_t size1,uintptr_t start2,size_t size2)810 overlaps(uintptr_t start1, size_t size1, uintptr_t start2, size_t size2)
811 {
812 	if (start1 < start2 + size2 &&
813 	    start1 + size1 >= start2) {
814 		printf("overlaps: %zx-%zx, %zx-%zx\n",
815 		    start1, start1 + size1, start2, start2 + size2);
816 		return (true);
817 	}
818 
819 	return (false);
820 }
821 #endif
822 
823 static int
multiboot2_exec(struct preloaded_file * fp)824 multiboot2_exec(struct preloaded_file *fp)
825 {
826 	multiboot2_info_header_t *mbi = NULL;
827 	struct preloaded_file *mfp;
828 	char *cmdline = NULL;
829 	struct devdesc *rootdev;
830 	struct file_metadata *md;
831 	int i, error, num;
832 	int rootfs = 0;
833 	size_t size;
834 	struct bios_smap *smap;
835 #if defined(EFI)
836 	multiboot_tag_module_t *module, *mp;
837 	struct relocator *relocator = NULL;
838 	EFI_MEMORY_DESCRIPTOR *map;
839 	UINTN map_size, desc_size;
840 	struct chunk_head *head;
841 	struct chunk *chunk;
842 	vm_offset_t tmp;
843 
844 	efi_getdev((void **)(&rootdev), NULL, NULL);
845 
846 	/*
847 	 * We need 5 pages for relocation. We'll allocate from the heap: while
848 	 * it's possible that our heap got placed low down enough to be in the
849 	 * way of where we're going to relocate our kernel, it's hopefully not
850 	 * likely.
851 	 */
852 	if ((relocator = malloc(EFI_PAGE_SIZE * 5)) == NULL) {
853 		printf("relocator malloc failed!\n");
854 		error = ENOMEM;
855 		goto error;
856 	}
857 
858 	if (overlaps((uintptr_t)relocator, EFI_PAGE_SIZE * 5,
859 	    load_addr, fp->f_size)) {
860 		printf("relocator pages overlap the kernel!\n");
861 		error = EINVAL;
862 		goto error;
863 	}
864 
865 #else
866 	i386_getdev((void **)(&rootdev), NULL, NULL);
867 
868 	if (have_framebuffer == false) {
869 		/* make sure we have text mode */
870 		bios_set_text_mode(VGA_TEXT_MODE);
871 	}
872 #endif
873 
874 	error = EINVAL;
875 	if (rootdev == NULL) {
876 		printf("can't determine root device\n");
877 		goto error;
878 	}
879 
880 	/*
881 	 * Set the image command line.
882 	 */
883 	if (fp->f_args == NULL) {
884 		cmdline = getenv("boot-args");
885 		if (cmdline != NULL) {
886 			fp->f_args = strdup(cmdline);
887 			if (fp->f_args == NULL) {
888 				error = ENOMEM;
889 				goto error;
890 			}
891 		}
892 	}
893 
894 	error = mb_kernel_cmdline(fp, rootdev, &cmdline);
895 	if (error != 0)
896 		goto error;
897 
898 	/* mb_kernel_cmdline() updates the environment. */
899 	build_environment_module();
900 
901 	/* Pass the loaded console font for kernel. */
902 	build_font_module();
903 
904 	size = mbi_size(fp, cmdline);	/* Get the size for MBI. */
905 
906 	/* Set up the base for mb_malloc. */
907 	i = 0;
908 	for (mfp = fp; mfp->f_next != NULL; mfp = mfp->f_next)
909 		i++;
910 
911 #if defined(EFI)
912 	/* We need space for kernel + MBI + # modules */
913 	num = (EFI_PAGE_SIZE - offsetof(struct relocator, rel_chunklist)) /
914 	    sizeof (struct chunk);
915 	if (i + 2 >= num) {
916 		printf("Too many modules, do not have space for relocator.\n");
917 		error = ENOMEM;
918 		goto error;
919 	}
920 
921 	last_addr = efi_loadaddr(LOAD_MEM, &size, mfp->f_addr + mfp->f_size);
922 	mbi = (multiboot2_info_header_t *)last_addr;
923 	if (mbi == NULL) {
924 		error = ENOMEM;
925 		goto error;
926 	}
927 	last_addr = (vm_offset_t)mbi->mbi_tags;
928 #else
929 	/* Start info block from the new page. */
930 	last_addr = i386_loadaddr(LOAD_MEM, &size, mfp->f_addr + mfp->f_size);
931 
932 	/* Do we have space for multiboot info? */
933 	if (last_addr + size >= memtop_copyin) {
934 		error = ENOMEM;
935 		goto error;
936 	}
937 
938 	mbi = (multiboot2_info_header_t *)PTOV(last_addr);
939 	last_addr = (vm_offset_t)mbi->mbi_tags;
940 #endif	/* EFI */
941 
942 	{
943 		multiboot_tag_string_t *tag;
944 		i = sizeof (multiboot_tag_string_t) + strlen(cmdline) + 1;
945 		tag = (multiboot_tag_string_t *)mb_malloc(i);
946 
947 		tag->mb_type = MULTIBOOT_TAG_TYPE_CMDLINE;
948 		tag->mb_size = i;
949 		memcpy(tag->mb_string, cmdline, strlen(cmdline) + 1);
950 		free(cmdline);
951 		cmdline = NULL;
952 	}
953 
954 	{
955 		multiboot_tag_string_t *tag;
956 		i = sizeof (multiboot_tag_string_t) + strlen(bootprog_info) + 1;
957 		tag = (multiboot_tag_string_t *)mb_malloc(i);
958 
959 		tag->mb_type = MULTIBOOT_TAG_TYPE_BOOT_LOADER_NAME;
960 		tag->mb_size = i;
961 		memcpy(tag->mb_string, bootprog_info,
962 		    strlen(bootprog_info) + 1);
963 	}
964 
965 #if !defined(EFI)
966 	/* Only set in case of BIOS. */
967 	{
968 		multiboot_tag_basic_meminfo_t *tag;
969 		tag = (multiboot_tag_basic_meminfo_t *)
970 		    mb_malloc(sizeof (*tag));
971 
972 		tag->mb_type = MULTIBOOT_TAG_TYPE_BASIC_MEMINFO;
973 		tag->mb_size = sizeof (*tag);
974 		tag->mb_mem_lower = bios_basemem / 1024;
975 		tag->mb_mem_upper = bios_extmem / 1024;
976 	}
977 #endif
978 
979 	num = 0;
980 	for (mfp = fp->f_next; mfp != NULL; mfp = mfp->f_next) {
981 		num++;
982 		if (mfp->f_type != NULL && strcmp(mfp->f_type, "rootfs") == 0)
983 			rootfs++;
984 	}
985 
986 	if (num == 0 || rootfs == 0) {
987 		/* We need at least one module - rootfs. */
988 		printf("No rootfs module provided, aborting\n");
989 		error = EINVAL;
990 		goto error;
991 	}
992 
993 	/*
994 	 * Set the stage for physical memory layout:
995 	 * - We have kernel at load_addr.
996 	 * - Modules are aligned to page boundary.
997 	 * - MBI is aligned to page boundary.
998 	 * - Set the tmp to point to physical address of the first module.
999 	 * - tmp != mfp->f_addr only in case of EFI.
1000 	 */
1001 #if defined(EFI)
1002 	tmp = roundup2(load_addr + fp->f_size + 1, MULTIBOOT_MOD_ALIGN);
1003 	module = (multiboot_tag_module_t *)last_addr;
1004 #endif
1005 
1006 	for (mfp = fp->f_next; mfp != NULL; mfp = mfp->f_next) {
1007 		multiboot_tag_module_t *tag;
1008 
1009 		num = strlen(mfp->f_name) + 1;
1010 		num += strlen(mfp->f_type) + 5 + 1;
1011 		if (mfp->f_args != NULL) {
1012 			num += strlen(mfp->f_args) + 1;
1013 		}
1014 		cmdline = malloc(num);
1015 		if (cmdline == NULL) {
1016 			error = ENOMEM;
1017 			goto error;
1018 		}
1019 
1020 		if (mfp->f_args != NULL)
1021 			snprintf(cmdline, num, "%s type=%s %s",
1022 			    mfp->f_name, mfp->f_type, mfp->f_args);
1023 		else
1024 			snprintf(cmdline, num, "%s type=%s",
1025 			    mfp->f_name, mfp->f_type);
1026 
1027 		tag = (multiboot_tag_module_t *)mb_malloc(sizeof (*tag) + num);
1028 
1029 		tag->mb_type = MULTIBOOT_TAG_TYPE_MODULE;
1030 		tag->mb_size = sizeof (*tag) + num;
1031 #if defined(EFI)
1032 		/*
1033 		 * We can assign module addresses only after BS have been
1034 		 * switched off.
1035 		 */
1036 		tag->mb_mod_start = 0;
1037 		tag->mb_mod_end = mfp->f_size;
1038 #else
1039 		tag->mb_mod_start = mfp->f_addr;
1040 		tag->mb_mod_end = mfp->f_addr + mfp->f_size;
1041 #endif
1042 		memcpy(tag->mb_cmdline, cmdline, num);
1043 		free(cmdline);
1044 		cmdline = NULL;
1045 	}
1046 
1047 	md = file_findmetadata(fp, MODINFOMD_SMAP);
1048 	if (md == NULL) {
1049 		printf("no memory smap\n");
1050 		error = EINVAL;
1051 		goto error;
1052 	}
1053 
1054 	smap = (struct bios_smap *)md->md_data;
1055 	num = md->md_size / sizeof (struct bios_smap); /* number of entries */
1056 
1057 	{
1058 		multiboot_tag_mmap_t *tag;
1059 		multiboot_mmap_entry_t *mmap_entry;
1060 
1061 		tag = (multiboot_tag_mmap_t *)
1062 		    mb_malloc(sizeof (*tag) +
1063 		    num * sizeof (multiboot_mmap_entry_t));
1064 
1065 		tag->mb_type = MULTIBOOT_TAG_TYPE_MMAP;
1066 		tag->mb_size = sizeof (*tag) +
1067 		    num * sizeof (multiboot_mmap_entry_t);
1068 		tag->mb_entry_size = sizeof (multiboot_mmap_entry_t);
1069 		tag->mb_entry_version = 0;
1070 		mmap_entry = (multiboot_mmap_entry_t *)tag->mb_entries;
1071 
1072 		for (i = 0; i < num; i++) {
1073 			mmap_entry[i].mmap_addr = smap[i].base;
1074 			mmap_entry[i].mmap_len = smap[i].length;
1075 			mmap_entry[i].mmap_type = smap[i].type;
1076 			mmap_entry[i].mmap_reserved = 0;
1077 		}
1078 	}
1079 
1080 	if (bootp_response != NULL) {
1081 		multiboot_tag_network_t *tag;
1082 		tag = (multiboot_tag_network_t *)
1083 		    mb_malloc(sizeof (*tag) + bootp_response_size);
1084 
1085 		tag->mb_type = MULTIBOOT_TAG_TYPE_NETWORK;
1086 		tag->mb_size = sizeof (*tag) + bootp_response_size;
1087 		memcpy(tag->mb_dhcpack, bootp_response, bootp_response_size);
1088 	}
1089 
1090 #if !defined(EFI)
1091 	multiboot_tag_vbe_t *tag;
1092 	extern multiboot_tag_vbe_t vbestate;
1093 
1094 	if (VBE_VALID_MODE(vbestate.vbe_mode)) {
1095 		tag = (multiboot_tag_vbe_t *)mb_malloc(sizeof (*tag));
1096 		memcpy(tag, &vbestate, sizeof (*tag));
1097 		tag->mb_type = MULTIBOOT_TAG_TYPE_VBE;
1098 		tag->mb_size = sizeof (*tag);
1099 	}
1100 #endif
1101 
1102 	if (rsdp != NULL) {
1103 		multiboot_tag_new_acpi_t *ntag;
1104 		multiboot_tag_old_acpi_t *otag;
1105 		uint32_t tsize;
1106 
1107 		if (rsdp->Revision == 0) {
1108 			tsize = sizeof (*otag) + sizeof (ACPI_RSDP_COMMON);
1109 			otag = (multiboot_tag_old_acpi_t *)mb_malloc(tsize);
1110 			otag->mb_type = MULTIBOOT_TAG_TYPE_ACPI_OLD;
1111 			otag->mb_size = tsize;
1112 			memcpy(otag->mb_rsdp, rsdp, sizeof (ACPI_RSDP_COMMON));
1113 		} else {
1114 			tsize = sizeof (*ntag) + rsdp->Length;
1115 			ntag = (multiboot_tag_new_acpi_t *)mb_malloc(tsize);
1116 			ntag->mb_type = MULTIBOOT_TAG_TYPE_ACPI_NEW;
1117 			ntag->mb_size = tsize;
1118 			memcpy(ntag->mb_rsdp, rsdp, rsdp->Length);
1119 		}
1120 	}
1121 
1122 #if defined(EFI)
1123 #ifdef  __LP64__
1124 	{
1125 		multiboot_tag_efi64_t *tag;
1126 		tag = (multiboot_tag_efi64_t *)
1127 		    mb_malloc(sizeof (*tag));
1128 
1129 		tag->mb_type = MULTIBOOT_TAG_TYPE_EFI64;
1130 		tag->mb_size = sizeof (*tag);
1131 		tag->mb_pointer = (uint64_t)(uintptr_t)ST;
1132 	}
1133 #else
1134 	{
1135 		multiboot_tag_efi32_t *tag;
1136 		tag = (multiboot_tag_efi32_t *)
1137 		    mb_malloc(sizeof (*tag));
1138 
1139 		tag->mb_type = MULTIBOOT_TAG_TYPE_EFI32;
1140 		tag->mb_size = sizeof (*tag);
1141 		tag->mb_pointer = (uint32_t)ST;
1142 	}
1143 #endif /* __LP64__ */
1144 #endif /* EFI */
1145 
1146 	if (have_framebuffer == true) {
1147 		multiboot_tag_framebuffer_t *tag;
1148 		extern multiboot_tag_framebuffer_t gfx_fb;
1149 #if defined(EFI)
1150 
1151 		tag = (multiboot_tag_framebuffer_t *)mb_malloc(sizeof (*tag));
1152 		memcpy(tag, &gfx_fb, sizeof (*tag));
1153 		tag->framebuffer_common.mb_type =
1154 		    MULTIBOOT_TAG_TYPE_FRAMEBUFFER;
1155 		tag->framebuffer_common.mb_size = sizeof (*tag);
1156 #else
1157 		extern multiboot_color_t *cmap;
1158 		uint32_t size;
1159 
1160 		if (gfx_fb.framebuffer_common.framebuffer_type ==
1161 		    MULTIBOOT_FRAMEBUFFER_TYPE_INDEXED) {
1162 			uint16_t nc;
1163 			nc = gfx_fb.u.fb1.framebuffer_palette_num_colors;
1164 			size = sizeof (struct multiboot_tag_framebuffer_common)
1165 			    + sizeof (nc)
1166 			    + nc * sizeof (multiboot_color_t);
1167 		} else {
1168 			size = sizeof (gfx_fb);
1169 		}
1170 
1171 		tag = (multiboot_tag_framebuffer_t *)mb_malloc(size);
1172 		memcpy(tag, &gfx_fb, sizeof (*tag));
1173 
1174 		tag->framebuffer_common.mb_type =
1175 		    MULTIBOOT_TAG_TYPE_FRAMEBUFFER;
1176 		tag->framebuffer_common.mb_size = size;
1177 
1178 		if (gfx_fb.framebuffer_common.framebuffer_type ==
1179 		    MULTIBOOT_FRAMEBUFFER_TYPE_INDEXED) {
1180 			gfx_fb.u.fb1.framebuffer_palette_num_colors = CMAP_SIZE;
1181 
1182 			memcpy(tag->u.fb1.framebuffer_palette, cmap,
1183 			    sizeof (multiboot_color_t) * CMAP_SIZE);
1184 		}
1185 #endif /* EFI */
1186 	}
1187 
1188 #if defined(EFI)
1189 	/* Leave EFI memmap last as we will also switch off the BS. */
1190 	{
1191 		multiboot_tag_efi_mmap_t *tag;
1192 		UINTN key;
1193 		EFI_STATUS status;
1194 
1195 		tag = (multiboot_tag_efi_mmap_t *)
1196 		    mb_malloc(sizeof (*tag));
1197 
1198 		map_size = 0;
1199 		status = BS->GetMemoryMap(&map_size,
1200 		    (EFI_MEMORY_DESCRIPTOR *)tag->mb_efi_mmap, &key,
1201 		    &desc_size, &tag->mb_descr_vers);
1202 		if (status != EFI_BUFFER_TOO_SMALL) {
1203 			error = EINVAL;
1204 			goto error;
1205 		}
1206 		map_size = roundup2(map_size, EFI_PAGE_SIZE);
1207 
1208 		i = 2;	/* Attempts to ExitBootServices() */
1209 		while (map_size <= efi_map_size && i > 0) {
1210 			status = BS->GetMemoryMap(&map_size,
1211 			    (EFI_MEMORY_DESCRIPTOR *)tag->mb_efi_mmap, &key,
1212 			    &desc_size, &tag->mb_descr_vers);
1213 			if (status == EFI_BUFFER_TOO_SMALL) {
1214 				/* Still too small? */
1215 				map_size += EFI_PAGE_SIZE;
1216 				continue;
1217 			}
1218 			if (EFI_ERROR(status)) {
1219 				error = EINVAL;
1220 				goto error;
1221 			}
1222 
1223 			if (keep_bs != 0)
1224 				break;
1225 
1226 			status = BS->ExitBootServices(IH, key);
1227 			if (status == EFI_SUCCESS) {
1228 				has_boot_services = false;
1229 				break;
1230 			}
1231 			i--;
1232 		}
1233 		if (status != EFI_SUCCESS) {
1234 			error = EINVAL;
1235 			goto error;
1236 		}
1237 
1238 		tag->mb_type = MULTIBOOT_TAG_TYPE_EFI_MMAP;
1239 		tag->mb_size = sizeof (*tag) + map_size;
1240 		tag->mb_descr_size = (uint32_t)desc_size;
1241 
1242 		map = (EFI_MEMORY_DESCRIPTOR *)tag->mb_efi_mmap;
1243 
1244 		last_addr += map_size;
1245 		last_addr = roundup2(last_addr, MULTIBOOT_TAG_ALIGN);
1246 	}
1247 #endif /* EFI */
1248 
1249 	/*
1250 	 * MB tag list end marker.
1251 	 */
1252 	{
1253 		multiboot_tag_t *tag = (multiboot_tag_t *)
1254 		    mb_malloc(sizeof (*tag));
1255 		tag->mb_type = MULTIBOOT_TAG_TYPE_END;
1256 		tag->mb_size = sizeof (*tag);
1257 	}
1258 
1259 	mbi->mbi_total_size = last_addr - (vm_offset_t)mbi;
1260 	mbi->mbi_reserved = 0;
1261 
1262 #if defined(EFI)
1263 	/*
1264 	 * At this point we have load_addr pointing to kernel load
1265 	 * address, module list in MBI having physical addresses,
1266 	 * module list in fp having logical addresses and tmp pointing to
1267 	 * physical address for MBI.
1268 	 * Now we must move all pieces to place and start the kernel.
1269 	 */
1270 	head = &relocator->rel_chunk_head;
1271 	STAILQ_INIT(head);
1272 
1273 	i = 0;
1274 	chunk = &relocator->rel_chunklist[i++];
1275 	chunk->chunk_vaddr = fp->f_addr;
1276 	chunk->chunk_paddr = load_addr;
1277 	chunk->chunk_size = fp->f_size;
1278 
1279 	STAILQ_INSERT_TAIL(head, chunk, chunk_next);
1280 
1281 	mp = module;
1282 	for (mfp = fp->f_next; mfp != NULL; mfp = mfp->f_next) {
1283 		chunk = &relocator->rel_chunklist[i++];
1284 		chunk->chunk_vaddr = mfp->f_addr;
1285 
1286 		/*
1287 		 * fix the mb_mod_start and mb_mod_end.
1288 		 */
1289 		mp->mb_mod_start = efi_physaddr(module, tmp, map,
1290 		    map_size / desc_size, desc_size, mfp->f_addr,
1291 		    mp->mb_mod_end);
1292 		if (mp->mb_mod_start == 0)
1293 			panic("Could not find memory for module");
1294 
1295 		mp->mb_mod_end += mp->mb_mod_start;
1296 		chunk->chunk_paddr = mp->mb_mod_start;
1297 		chunk->chunk_size = mfp->f_size;
1298 		STAILQ_INSERT_TAIL(head, chunk, chunk_next);
1299 
1300 		mp = (multiboot_tag_module_t *)
1301 		    roundup2((uintptr_t)mp + mp->mb_size,
1302 		    MULTIBOOT_TAG_ALIGN);
1303 	}
1304 	chunk = &relocator->rel_chunklist[i++];
1305 	chunk->chunk_vaddr = (EFI_VIRTUAL_ADDRESS)(uintptr_t)mbi;
1306 	chunk->chunk_paddr = efi_physaddr(module, tmp, map,
1307 	    map_size / desc_size, desc_size, (uintptr_t)mbi,
1308 	    mbi->mbi_total_size);
1309 	chunk->chunk_size = mbi->mbi_total_size;
1310 	STAILQ_INSERT_TAIL(head, chunk, chunk_next);
1311 
1312 	trampoline = (void *)(uintptr_t)relocator + EFI_PAGE_SIZE;
1313 	memmove(trampoline, multiboot_tramp, EFI_PAGE_SIZE);
1314 
1315 	relocator->rel_copy = (uintptr_t)trampoline + EFI_PAGE_SIZE;
1316 	memmove((void *)relocator->rel_copy, efi_copy_finish, EFI_PAGE_SIZE);
1317 
1318 	relocator->rel_memmove = (uintptr_t)relocator->rel_copy + EFI_PAGE_SIZE;
1319 	memmove((void *)relocator->rel_memmove, memmove, EFI_PAGE_SIZE);
1320 	relocator->rel_stack = relocator->rel_memmove + EFI_PAGE_SIZE - 8;
1321 
1322 	trampoline(MULTIBOOT2_BOOTLOADER_MAGIC, relocator, entry_addr);
1323 #else
1324 	dev_cleanup();
1325 	__exec((void *)VTOP(multiboot_tramp), MULTIBOOT2_BOOTLOADER_MAGIC,
1326 	    (void *)entry_addr, (void *)VTOP(mbi));
1327 #endif /* EFI */
1328 	panic("exec returned");
1329 
1330 error:
1331 	free(cmdline);
1332 
1333 #if defined(EFI)
1334 	free(relocator);
1335 
1336 	if (mbi != NULL)
1337 		efi_free_loadaddr((vm_offset_t)mbi, EFI_SIZE_TO_PAGES(size));
1338 #endif
1339 
1340 	return (error);
1341 }
1342