xref: /illumos-gate/usr/src/boot/common/multiboot2.c (revision b72c8d00)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2017 Toomas Soome <tsoome@me.com>
14  * Copyright 2019, Joyent, Inc.
15  */
16 
17 /*
18  * This module adds support for loading and booting illumos multiboot2
19  * kernel. This code is only built to support the illumos kernel, it does
20  * not support xen.
21  */
22 
23 #include <sys/cdefs.h>
24 #include <sys/stddef.h>
25 
26 #include <sys/param.h>
27 #include <sys/exec.h>
28 #include <sys/linker.h>
29 #include <sys/module.h>
30 #include <sys/stdint.h>
31 #include <sys/multiboot2.h>
32 #include <stand.h>
33 #include <stdbool.h>
34 #include <machine/elf.h>
35 #include "libzfs.h"
36 
37 #include "bootstrap.h"
38 #include <sys/consplat.h>
39 
40 #include <machine/metadata.h>
41 #include <machine/pc/bios.h>
42 
43 #define	SUPPORT_DHCP
44 #include <bootp.h>
45 
46 #if !defined(EFI)
47 #include "../i386/btx/lib/btxv86.h"
48 #include "libi386.h"
49 #include "vbe.h"
50 
51 #else
52 #include <efi.h>
53 #include <efilib.h>
54 #include "loader_efi.h"
55 
56 static void (*trampoline)(uint32_t, struct relocator *, uint64_t);
57 static UINTN efi_map_size;		/* size of efi memory map */
58 #endif
59 
60 #include "platform/acfreebsd.h"
61 #include "acconfig.h"
62 #define	ACPI_SYSTEM_XFACE
63 #include "actypes.h"
64 #include "actbl.h"
65 
66 extern ACPI_TABLE_RSDP *rsdp;
67 
68 /* MB data heap pointer. */
69 static vm_offset_t last_addr;
70 
71 static int multiboot2_loadfile(char *, uint64_t, struct preloaded_file **);
72 static int multiboot2_exec(struct preloaded_file *);
73 
74 struct file_format multiboot2 = { multiboot2_loadfile, multiboot2_exec };
75 static bool keep_bs = false;
76 static bool have_framebuffer = false;
77 static vm_offset_t load_addr;
78 static vm_offset_t entry_addr;
79 bool has_boot_services = true;
80 
81 /*
82  * Validate tags in info request. This function is provided just to
83  * recognize the current tag list and only serves as a limited
84  * safe guard against possibly corrupt information.
85  */
86 static bool
is_info_request_valid(multiboot_header_tag_information_request_t * rtag)87 is_info_request_valid(multiboot_header_tag_information_request_t *rtag)
88 {
89 	int i;
90 
91 	/*
92 	 * If the tag is optional and we do not support it, we do not
93 	 * have to do anything special, so we skip optional tags.
94 	 */
95 	if (rtag->mbh_flags & MULTIBOOT_HEADER_TAG_OPTIONAL)
96 		return (true);
97 
98 	for (i = 0; i < (rtag->mbh_size - sizeof (*rtag)) /
99 	    sizeof (rtag->mbh_requests[0]); i++)
100 		switch (rtag->mbh_requests[i]) {
101 		case MULTIBOOT_TAG_TYPE_END:
102 		case MULTIBOOT_TAG_TYPE_CMDLINE:
103 		case MULTIBOOT_TAG_TYPE_BOOT_LOADER_NAME:
104 		case MULTIBOOT_TAG_TYPE_MODULE:
105 		case MULTIBOOT_TAG_TYPE_BASIC_MEMINFO:
106 		case MULTIBOOT_TAG_TYPE_BOOTDEV:
107 		case MULTIBOOT_TAG_TYPE_MMAP:
108 		case MULTIBOOT_TAG_TYPE_FRAMEBUFFER:
109 		case MULTIBOOT_TAG_TYPE_VBE:
110 		case MULTIBOOT_TAG_TYPE_ELF_SECTIONS:
111 		case MULTIBOOT_TAG_TYPE_APM:
112 		case MULTIBOOT_TAG_TYPE_EFI32:
113 		case MULTIBOOT_TAG_TYPE_EFI64:
114 		case MULTIBOOT_TAG_TYPE_ACPI_OLD:
115 		case MULTIBOOT_TAG_TYPE_ACPI_NEW:
116 		case MULTIBOOT_TAG_TYPE_NETWORK:
117 		case MULTIBOOT_TAG_TYPE_EFI_MMAP:
118 		case MULTIBOOT_TAG_TYPE_EFI_BS:
119 		case MULTIBOOT_TAG_TYPE_EFI32_IH:
120 		case MULTIBOOT_TAG_TYPE_EFI64_IH:
121 		case MULTIBOOT_TAG_TYPE_LOAD_BASE_ADDR:
122 			break;
123 		default:
124 			printf("unsupported information tag: 0x%x\n",
125 			    rtag->mbh_requests[i]);
126 			return (false);
127 		}
128 	return (true);
129 }
130 
131 static int
multiboot2_loadfile(char * filename,uint64_t dest,struct preloaded_file ** result)132 multiboot2_loadfile(char *filename, uint64_t dest,
133     struct preloaded_file **result)
134 {
135 	int fd, error;
136 	uint32_t i;
137 	struct stat st;
138 	caddr_t header_search;
139 	multiboot2_header_t *header;
140 	multiboot_header_tag_t *tag;
141 	multiboot_header_tag_address_t *addr_tag = NULL;
142 	multiboot_header_tag_entry_address_t *entry_tag = NULL;
143 	struct preloaded_file *fp;
144 
145 	/* This allows to check other file formats from file_formats array. */
146 	error = EFTYPE;
147 	if (filename == NULL)
148 		return (error);
149 
150 	/* is kernel already loaded? */
151 	fp = file_findfile(NULL, NULL);
152 	if (fp != NULL)
153 		return (error);
154 
155 	if ((fd = open(filename, O_RDONLY)) == -1)
156 		return (errno);
157 
158 	/*
159 	 * Read MULTIBOOT_SEARCH size in order to search for the
160 	 * multiboot magic header.
161 	 */
162 	header_search = malloc(MULTIBOOT_SEARCH);
163 	if (header_search == NULL) {
164 		close(fd);
165 		return (ENOMEM);
166 	}
167 
168 	if (read(fd, header_search, MULTIBOOT_SEARCH) != MULTIBOOT_SEARCH)
169 		goto out;
170 
171 	header = NULL;
172 	for (i = 0; i <= (MULTIBOOT_SEARCH - sizeof (multiboot2_header_t));
173 	    i += MULTIBOOT_HEADER_ALIGN) {
174 		header = (multiboot2_header_t *)(header_search + i);
175 
176 		/* Do we have match on magic? */
177 		if (header->mb2_magic != MULTIBOOT2_HEADER_MAGIC) {
178 			header = NULL;
179 			continue;
180 		}
181 		/*
182 		 * Validate checksum, the sum of magic + architecture +
183 		 * header_length + checksum must equal 0.
184 		 */
185 		if (header->mb2_magic + header->mb2_architecture +
186 		    header->mb2_header_length + header->mb2_checksum != 0) {
187 			header = NULL;
188 			continue;
189 		}
190 		/*
191 		 * Finally, the entire header must fit within MULTIBOOT_SEARCH.
192 		 */
193 		if (i + header->mb2_header_length > MULTIBOOT_SEARCH) {
194 			header = NULL;
195 			continue;
196 		}
197 		break;
198 	}
199 
200 	if (header == NULL)
201 		goto out;
202 
203 	have_framebuffer = false;
204 	for (tag = header->mb2_tags; tag->mbh_type != MULTIBOOT_TAG_TYPE_END;
205 	    tag = (multiboot_header_tag_t *)((uintptr_t)tag +
206 	    roundup2(tag->mbh_size, MULTIBOOT_TAG_ALIGN))) {
207 		switch (tag->mbh_type) {
208 		case MULTIBOOT_HEADER_TAG_INFORMATION_REQUEST:
209 			if (is_info_request_valid((void*)tag) == false)
210 				goto out;
211 			break;
212 		case MULTIBOOT_HEADER_TAG_ADDRESS:
213 			addr_tag = (multiboot_header_tag_address_t *)tag;
214 			break;
215 		case MULTIBOOT_HEADER_TAG_ENTRY_ADDRESS:
216 			entry_tag =
217 			    (multiboot_header_tag_entry_address_t *)tag;
218 			break;
219 		case MULTIBOOT_HEADER_TAG_CONSOLE_FLAGS:
220 			break;
221 		case MULTIBOOT_HEADER_TAG_FRAMEBUFFER:
222 			have_framebuffer = true;
223 			break;
224 		case MULTIBOOT_HEADER_TAG_MODULE_ALIGN:
225 			/* we always align modules */
226 			break;
227 		case MULTIBOOT_HEADER_TAG_EFI_BS:
228 			keep_bs = true;
229 			break;
230 		default:
231 			if (!(tag->mbh_flags & MULTIBOOT_HEADER_TAG_OPTIONAL)) {
232 				printf("unsupported tag: 0x%x\n",
233 				    tag->mbh_type);
234 				goto out;
235 			}
236 		}
237 	}
238 
239 	/*
240 	 * We must have addr_tag and entry_tag to load a 64-bit kernel.
241 	 * If these tags are missing, we either have a 32-bit kernel, or
242 	 * this is not our kernel at all.
243 	 */
244 	if (addr_tag != NULL && entry_tag != NULL) {
245 		fp = file_alloc();
246 		if (fp == NULL) {
247 			error = ENOMEM;
248 			goto out;
249 		}
250 		if (lseek(fd, 0, SEEK_SET) == -1) {
251 			printf("lseek failed\n");
252 			error = EIO;
253 			file_discard(fp);
254 			goto out;
255 		}
256 		if (fstat(fd, &st) < 0) {
257 			printf("fstat failed\n");
258 			error = EIO;
259 			file_discard(fp);
260 			goto out;
261 		}
262 
263 		load_addr = addr_tag->mbh_load_addr;
264 		entry_addr = entry_tag->mbh_entry_addr;
265 		fp->f_addr = archsw.arch_loadaddr(LOAD_KERN, filename,
266 		    addr_tag->mbh_load_addr);
267 		if (fp->f_addr == 0) {
268 			error = ENOMEM;
269 			file_discard(fp);
270 			goto out;
271 		}
272 		fp->f_size = archsw.arch_readin(fd, fp->f_addr, st.st_size);
273 
274 		if (fp->f_size != st.st_size) {
275 			printf("error reading %s: %s\n", filename,
276 			    strerror(errno));
277 			file_discard(fp);
278 			error = EIO;
279 			goto out;
280 		}
281 
282 		fp->f_name = strdup(filename);
283 		fp->f_type = strdup("aout multiboot2 kernel");
284 		if (fp->f_name == NULL || fp->f_type == NULL) {
285 			error = ENOMEM;
286 			file_discard(fp);
287 			goto out;
288 		}
289 
290 		fp->f_metadata = NULL;
291 		error = 0;
292 	} else {
293 #if defined(EFI)
294 		/* 32-bit kernel is not yet supported for EFI */
295 		printf("32-bit kernel is not supported by UEFI loader\n");
296 		error = ENOTSUP;
297 		goto out;
298 #endif
299 		/* elf32_loadfile_raw will fill the attributes in fp. */
300 		error = elf32_loadfile_raw(filename, dest, &fp, 2);
301 		if (error != 0) {
302 			printf("elf32_loadfile_raw failed: %d unable to "
303 			    "load multiboot2 kernel\n", error);
304 			goto out;
305 		}
306 		entry_addr = fp->f_addr;
307 		/*
308 		 * We want the load_addr to have some legal value,
309 		 * so we set it same as the entry_addr.
310 		 * The distinction is important with UEFI, but not
311 		 * with BIOS version, because BIOS version does not use
312 		 * staging area.
313 		 */
314 		load_addr = fp->f_addr;
315 	}
316 
317 	setenv("kernelname", fp->f_name, 1);
318 #if defined(EFI)
319 	efi_addsmapdata(fp);
320 #else
321 	bios_addsmapdata(fp);
322 #endif
323 	*result = fp;
324 out:
325 	free(header_search);
326 	close(fd);
327 	return (error);
328 }
329 
330 /*
331  * Search the command line for named property.
332  *
333  * Return codes:
334  *	0	The name is found, we return the data in value and len.
335  *	ENOENT	The name is not found.
336  *	EINVAL	The provided command line is badly formed.
337  */
338 static int
find_property_value(const char * cmd,const char * name,const char ** value,size_t * len)339 find_property_value(const char *cmd, const char *name, const char **value,
340     size_t *len)
341 {
342 	const char *namep, *valuep;
343 	size_t name_len, value_len;
344 	int quoted;
345 
346 	*value = NULL;
347 	*len = 0;
348 
349 	if (cmd == NULL)
350 		return (ENOENT);
351 
352 	while (*cmd != '\0') {
353 		if (cmd[0] != '-' || cmd[1] != 'B') {
354 			cmd++;
355 			continue;
356 		}
357 		cmd += 2;	/* Skip -B */
358 		while (cmd[0] == ' ' || cmd[0] == '\t')
359 			cmd++;	/* Skip whitespaces. */
360 		while (*cmd != '\0' && cmd[0] != ' ' && cmd[0] != '\t') {
361 			namep = cmd;
362 			valuep = strchr(cmd, '=');
363 			if (valuep == NULL)
364 				break;
365 			name_len = valuep - namep;
366 			valuep++;
367 			value_len = 0;
368 			quoted = 0;
369 			for (;; ++value_len) {
370 				if (valuep[value_len] == '\0')
371 					break;
372 
373 				/* Is this value quoted? */
374 				if (value_len == 0 &&
375 				    (valuep[0] == '\'' || valuep[0] == '"')) {
376 					quoted = valuep[0];
377 					++value_len;
378 				}
379 
380 				/*
381 				 * In the quote accept any character,
382 				 * but look for ending quote.
383 				 */
384 				if (quoted != 0) {
385 					if (valuep[value_len] == quoted)
386 						quoted = 0;
387 					continue;
388 				}
389 
390 				/* A comma or white space ends the value. */
391 				if (valuep[value_len] == ',' ||
392 				    valuep[value_len] == ' ' ||
393 				    valuep[value_len] == '\t')
394 					break;
395 			}
396 			if (quoted != 0) {
397 				printf("Missing closing '%c' in \"%s\"\n",
398 				    quoted, valuep);
399 				return (EINVAL);
400 			}
401 			if (value_len != 0) {
402 				if (strncmp(namep, name, name_len) == 0) {
403 					*value = valuep;
404 					*len = value_len;
405 					return (0);
406 				}
407 			}
408 			cmd = valuep + value_len;
409 			while (*cmd == ',')
410 				cmd++;
411 		}
412 	}
413 	return (ENOENT);
414 }
415 
416 /*
417  * If command line has " -B ", insert property after "-B ", otherwise
418  * append to command line.
419  */
420 static char *
insert_cmdline(const char * head,const char * prop)421 insert_cmdline(const char *head, const char *prop)
422 {
423 	const char *prop_opt = " -B ";
424 	char *cmdline, *tail;
425 	int len = 0;
426 
427 	tail = strstr(head, prop_opt);
428 	if (tail != NULL) {
429 		ptrdiff_t diff;
430 		tail += strlen(prop_opt);
431 		diff = tail - head;
432 		if (diff >= INT_MAX)
433 			return (NULL);
434 		len = (int)diff;
435 	}
436 
437 	if (tail == NULL)
438 		asprintf(&cmdline, "%s%s%s", head, prop_opt, prop);
439 	else
440 		asprintf(&cmdline, "%.*s%s,%s", len, head, prop, tail);
441 
442 	return (cmdline);
443 }
444 
445 /*
446  * Since we have no way to pass the environment to the mb1 kernel other than
447  * through arguments, we need to take care of console setup.
448  *
449  * If the console is in mirror mode, set the kernel console from $os_console.
450  * If it's unset, use first item from $console.
451  * If $console is "ttyX", also pass $ttyX-mode, since it may have been set by
452  * the user.
453  *
454  * In case of memory allocation errors, just return the original command line
455  * so we have a chance of booting.
456  *
457  * On success, cl will be freed and a new, allocated command line string is
458  * returned.
459  *
460  * For the mb2 kernel, we only set command line console if os_console is set.
461  * We can not overwrite console in the environment, as it can disrupt the
462  * loader console messages, and we do not want to deal with the os_console
463  * in the kernel.
464  */
465 static char *
update_cmdline(char * cl,bool mb2)466 update_cmdline(char *cl, bool mb2)
467 {
468 	char *os_console = getenv("os_console");
469 	char *ttymode = NULL;
470 	char mode[10];
471 	char *tmp;
472 	const char *prop;
473 	size_t plen;
474 	int rv;
475 
476 	if (mb2 == true && os_console == NULL)
477 		return (cl);
478 
479 	if (os_console == NULL) {
480 		tmp = strdup(getenv("console"));
481 		os_console = strsep(&tmp, ", ");
482 	} else {
483 		os_console = strdup(os_console);
484 	}
485 
486 	if (os_console == NULL)
487 		return (cl);
488 
489 	if (mb2 == false && strncmp(os_console, "tty", 3) == 0) {
490 		snprintf(mode, sizeof (mode), "%s-mode", os_console);
491 		/*
492 		 * The ttyX-mode variable is set by our serial console
493 		 * driver for ttya-ttyd. However, since the os_console
494 		 * values are not verified, it is possible we get bogus
495 		 * name and no mode variable. If so, we do not set console
496 		 * property and let the kernel use defaults.
497 		 */
498 		if ((ttymode = getenv(mode)) == NULL)
499 			return (cl);
500 	}
501 
502 	rv = find_property_value(cl, "console", &prop, &plen);
503 	if (rv != 0 && rv != ENOENT) {
504 		free(os_console);
505 		return (cl);
506 	}
507 
508 	/* If console is set and this is MB2 boot, we are done. */
509 	if (rv == 0 && mb2 == true) {
510 		free(os_console);
511 		return (cl);
512 	}
513 
514 	/* If console is set, do we need to set tty mode? */
515 	if (rv == 0) {
516 		const char *ttyp = NULL;
517 		size_t ttylen;
518 
519 		free(os_console);
520 		os_console = NULL;
521 		*mode = '\0';
522 		if (strncmp(prop, "tty", 3) == 0 && plen == 4) {
523 			strncpy(mode, prop, plen);
524 			mode[plen] = '\0';
525 			strncat(mode, "-mode", 5);
526 			find_property_value(cl, mode, &ttyp, &ttylen);
527 		}
528 
529 		if (*mode != '\0' && ttyp == NULL)
530 			ttymode = getenv(mode);
531 		else
532 			return (cl);
533 	}
534 
535 	/* Build updated command line. */
536 	if (os_console != NULL) {
537 		char *propstr;
538 
539 		asprintf(&propstr, "console=%s", os_console);
540 		free(os_console);
541 		if (propstr == NULL) {
542 			return (cl);
543 		}
544 
545 		tmp = insert_cmdline(cl, propstr);
546 		free(propstr);
547 		if (tmp == NULL)
548 			return (cl);
549 
550 		free(cl);
551 		cl = tmp;
552 	}
553 	if (ttymode != NULL) {
554 		char *propstr;
555 
556 		asprintf(&propstr, "%s=\"%s\"", mode, ttymode);
557 		if (propstr == NULL)
558 			return (cl);
559 
560 		tmp = insert_cmdline(cl, propstr);
561 		free(propstr);
562 		if (tmp == NULL)
563 			return (cl);
564 		free(cl);
565 		cl = tmp;
566 	}
567 
568 	return (cl);
569 }
570 
571 /*
572  * Build the kernel command line. Shared function between MB1 and MB2.
573  *
574  * In both cases, if fstype is set and is not zfs, we do not set up
575  * zfs-bootfs property. But we set kernel file name and options.
576  *
577  * For the MB1, we only can pass properties on command line, so
578  * we will set console, ttyX-mode (for serial console) and zfs-bootfs.
579  *
580  * For the MB2, we can pass properties in environment, but if os_console
581  * is set in environment, we need to add console property on the kernel
582  * command line.
583  *
584  * The console properties are managed in update_cmdline().
585  */
586 int
mb_kernel_cmdline(struct preloaded_file * fp,struct devdesc * rootdev,char ** line)587 mb_kernel_cmdline(struct preloaded_file *fp, struct devdesc *rootdev,
588     char **line)
589 {
590 	const char *fs = getenv("fstype");
591 	char *cmdline;
592 	size_t len;
593 	bool zfs_root = false;
594 	bool mb2;
595 	int rv;
596 
597 	/*
598 	 * With multiple console devices and "os_console" variable not
599 	 * set, set os_console to last input device.
600 	 */
601 	rv = cons_inputdev();
602 	if (rv != -1)
603 		(void) setenv("os_console", consoles[rv]->c_name, 0);
604 
605 	/*
606 	 * 64-bit kernel has aout header, 32-bit kernel is elf, and the
607 	 * type strings are different. Lets just search for "multiboot2".
608 	 */
609 	if (strstr(fp->f_type, "multiboot2") == NULL)
610 		mb2 = false;
611 	else
612 		mb2 = true;
613 
614 	if (rootdev->d_dev->dv_type == DEVT_ZFS)
615 		zfs_root = true;
616 
617 	/* If we have fstype set in env, reset zfs_root if needed. */
618 	if (fs != NULL && strcmp(fs, "zfs") != 0)
619 		zfs_root = false;
620 
621 	/*
622 	 * If we have fstype set on the command line,
623 	 * reset zfs_root if needed.
624 	 */
625 	rv = find_property_value(fp->f_args, "fstype", &fs, &len);
626 	if (rv != 0 && rv != ENOENT)
627 		return (rv);
628 
629 	if (fs != NULL && strncmp(fs, "zfs", len) != 0)
630 		zfs_root = false;
631 
632 	/* zfs_bootfs() will set the environment, it must be called. */
633 	if (zfs_root == true)
634 		fs = zfs_bootfs(rootdev);
635 
636 	if (fp->f_args == NULL)
637 		cmdline = strdup(fp->f_name);
638 	else
639 		asprintf(&cmdline, "%s %s", fp->f_name, fp->f_args);
640 
641 	if (cmdline == NULL)
642 		return (ENOMEM);
643 
644 	/* Append zfs-bootfs for MB1 command line. */
645 	if (mb2 == false && zfs_root == true) {
646 		char *tmp;
647 
648 		tmp = insert_cmdline(cmdline, fs);
649 		free(cmdline);
650 		if (tmp == NULL)
651 			return (ENOMEM);
652 		cmdline = tmp;
653 	}
654 
655 	*line = update_cmdline(cmdline, mb2);
656 	return (0);
657 }
658 
659 /*
660  * Returns allocated virtual address from MB info area.
661  */
662 static vm_offset_t
mb_malloc(size_t n)663 mb_malloc(size_t n)
664 {
665 	vm_offset_t ptr = last_addr;
666 	last_addr = roundup(last_addr + n, MULTIBOOT_TAG_ALIGN);
667 	return (ptr);
668 }
669 
670 /*
671  * Calculate size for module tag list.
672  */
673 static size_t
module_size(struct preloaded_file * fp)674 module_size(struct preloaded_file *fp)
675 {
676 	size_t len, size;
677 	struct preloaded_file *mfp;
678 
679 	size = 0;
680 	for (mfp = fp->f_next; mfp != NULL; mfp = mfp->f_next) {
681 		len = strlen(mfp->f_name) + 1;
682 		len += strlen(mfp->f_type) + 5 + 1; /* 5 is for "type=" */
683 		if (mfp->f_args != NULL)
684 			len += strlen(mfp->f_args) + 1;
685 		size += sizeof (multiboot_tag_module_t) + len;
686 		size = roundup(size, MULTIBOOT_TAG_ALIGN);
687 	}
688 	return (size);
689 }
690 
691 #if defined(EFI)
692 /*
693  * Calculate size for UEFI memory map tag.
694  */
695 #define	EFI_EXTRA_PAGES	3
696 
697 static int
efimemmap_size(void)698 efimemmap_size(void)
699 {
700 	UINTN size, cur_size, desc_size;
701 	EFI_MEMORY_DESCRIPTOR *mmap;
702 	EFI_STATUS ret;
703 
704 	size = EFI_PAGE_SIZE;		/* Start with 4k. */
705 	while (1) {
706 		cur_size = size;
707 		mmap = malloc(cur_size);
708 		if (mmap == NULL)
709 			return (0);
710 		ret = BS->GetMemoryMap(&cur_size, mmap, NULL, &desc_size, NULL);
711 		free(mmap);
712 		if (ret == EFI_SUCCESS)
713 			break;
714 		if (ret == EFI_BUFFER_TOO_SMALL) {
715 			if (size < cur_size)
716 				size = cur_size;
717 			size += (EFI_PAGE_SIZE);
718 		} else
719 			return (0);
720 	}
721 
722 	/* EFI MMAP will grow when we allocate MBI, set some buffer. */
723 	size += (EFI_EXTRA_PAGES << EFI_PAGE_SHIFT);
724 	size = roundup2(size, EFI_PAGE_SIZE);
725 	efi_map_size = size;	/* Record the calculated size. */
726 	return (sizeof (multiboot_tag_efi_mmap_t) + size);
727 }
728 #endif
729 
730 /*
731  * Calculate size for bios smap tag.
732  */
733 static size_t
biossmap_size(struct preloaded_file * fp)734 biossmap_size(struct preloaded_file *fp)
735 {
736 	int num;
737 	struct file_metadata *md;
738 
739 	md = file_findmetadata(fp, MODINFOMD_SMAP);
740 	if (md == NULL)
741 		return (0);
742 
743 	num = md->md_size / sizeof (struct bios_smap); /* number of entries */
744 	return (sizeof (multiboot_tag_mmap_t) +
745 	    num * sizeof (multiboot_mmap_entry_t));
746 }
747 
748 static size_t
mbi_size(struct preloaded_file * fp,char * cmdline)749 mbi_size(struct preloaded_file *fp, char *cmdline)
750 {
751 	size_t size;
752 #if !defined(EFI)
753 	extern multiboot_tag_framebuffer_t gfx_fb;
754 #endif
755 
756 	size = sizeof (uint32_t) * 2; /* first 2 fields from MBI header */
757 	size += sizeof (multiboot_tag_string_t) + strlen(cmdline) + 1;
758 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
759 	size += sizeof (multiboot_tag_string_t) + strlen(bootprog_info) + 1;
760 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
761 #if !defined(EFI)
762 	size += sizeof (multiboot_tag_basic_meminfo_t);
763 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
764 #endif
765 	size += module_size(fp);
766 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
767 #if defined(EFI)
768 	size += sizeof (multiboot_tag_efi64_t);
769 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
770 	size += efimemmap_size();
771 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
772 
773 	if (have_framebuffer == true) {
774 		size += sizeof (multiboot_tag_framebuffer_t);
775 		size = roundup2(size, MULTIBOOT_TAG_ALIGN);
776 	}
777 #endif
778 
779 	size += biossmap_size(fp);
780 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
781 
782 #if !defined(EFI)
783 	if (gfx_fb.framebuffer_common.framebuffer_type ==
784 	    MULTIBOOT_FRAMEBUFFER_TYPE_INDEXED) {
785 		size += sizeof (struct multiboot_tag_framebuffer_common);
786 		size += CMAP_SIZE * sizeof (multiboot_color_t);
787 	} else {
788 		size += sizeof (multiboot_tag_framebuffer_t);
789 	}
790 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
791 
792 	size += sizeof (multiboot_tag_vbe_t);
793 	size = roundup2(size, MULTIBOOT_TAG_ALIGN);
794 #endif
795 
796 	if (bootp_response != NULL) {
797 		size += sizeof (multiboot_tag_network_t) + bootp_response_size;
798 		size = roundup2(size, MULTIBOOT_TAG_ALIGN);
799 	}
800 
801 	if (rsdp != NULL) {
802 		if (rsdp->Revision == 0) {
803 			size += sizeof (multiboot_tag_old_acpi_t) +
804 			    sizeof (ACPI_RSDP_COMMON);
805 		} else {
806 			size += sizeof (multiboot_tag_new_acpi_t) +
807 			    rsdp->Length;
808 		}
809 		size = roundup2(size, MULTIBOOT_TAG_ALIGN);
810 	}
811 	size += sizeof (multiboot_tag_t);
812 
813 	return (size);
814 }
815 
816 #if defined(EFI)
817 static bool
overlaps(uintptr_t start1,size_t size1,uintptr_t start2,size_t size2)818 overlaps(uintptr_t start1, size_t size1, uintptr_t start2, size_t size2)
819 {
820 	if (start1 < start2 + size2 &&
821 	    start1 + size1 >= start2) {
822 		printf("overlaps: %zx-%zx, %zx-%zx\n",
823 		    start1, start1 + size1, start2, start2 + size2);
824 		return (true);
825 	}
826 
827 	return (false);
828 }
829 #endif
830 
831 static int
multiboot2_exec(struct preloaded_file * fp)832 multiboot2_exec(struct preloaded_file *fp)
833 {
834 	multiboot2_info_header_t *mbi = NULL;
835 	struct preloaded_file *mfp;
836 	char *cmdline = NULL;
837 	struct devdesc *rootdev;
838 	struct file_metadata *md;
839 	int i, error, num;
840 	int rootfs = 0;
841 	size_t size;
842 	struct bios_smap *smap;
843 #if defined(EFI)
844 	multiboot_tag_module_t *module, *mp;
845 	struct relocator *relocator = NULL;
846 	EFI_MEMORY_DESCRIPTOR *map;
847 	UINTN map_size, desc_size;
848 	struct chunk_head *head;
849 	struct chunk *chunk;
850 	vm_offset_t tmp;
851 
852 	efi_getdev((void **)(&rootdev), NULL, NULL);
853 
854 	/*
855 	 * We need 5 pages for relocation. We'll allocate from the heap: while
856 	 * it's possible that our heap got placed low down enough to be in the
857 	 * way of where we're going to relocate our kernel, it's hopefully not
858 	 * likely.
859 	 */
860 	if ((relocator = malloc(EFI_PAGE_SIZE * 5)) == NULL) {
861 		printf("relocator malloc failed!\n");
862 		error = ENOMEM;
863 		goto error;
864 	}
865 
866 	if (overlaps((uintptr_t)relocator, EFI_PAGE_SIZE * 5,
867 	    load_addr, fp->f_size)) {
868 		printf("relocator pages overlap the kernel!\n");
869 		error = EINVAL;
870 		goto error;
871 	}
872 
873 #else
874 	i386_getdev((void **)(&rootdev), NULL, NULL);
875 
876 	if (have_framebuffer == false) {
877 		/* make sure we have text mode */
878 		bios_set_text_mode(VGA_TEXT_MODE);
879 	}
880 #endif
881 
882 	error = EINVAL;
883 	if (rootdev == NULL) {
884 		printf("can't determine root device\n");
885 		goto error;
886 	}
887 
888 	/*
889 	 * Set the image command line.
890 	 */
891 	if (fp->f_args == NULL) {
892 		cmdline = getenv("boot-args");
893 		if (cmdline != NULL) {
894 			fp->f_args = strdup(cmdline);
895 			if (fp->f_args == NULL) {
896 				error = ENOMEM;
897 				goto error;
898 			}
899 		}
900 	}
901 
902 	error = mb_kernel_cmdline(fp, rootdev, &cmdline);
903 	if (error != 0)
904 		goto error;
905 
906 	/* mb_kernel_cmdline() updates the environment. */
907 	build_environment_module();
908 
909 	/* Pass the loaded console font for kernel. */
910 	build_font_module();
911 
912 	size = mbi_size(fp, cmdline);	/* Get the size for MBI. */
913 
914 	/* Set up the base for mb_malloc. */
915 	i = 0;
916 	for (mfp = fp; mfp->f_next != NULL; mfp = mfp->f_next)
917 		i++;
918 
919 #if defined(EFI)
920 	/* We need space for kernel + MBI + # modules */
921 	num = (EFI_PAGE_SIZE - offsetof(struct relocator, rel_chunklist)) /
922 	    sizeof (struct chunk);
923 	if (i + 2 >= num) {
924 		printf("Too many modules, do not have space for relocator.\n");
925 		error = ENOMEM;
926 		goto error;
927 	}
928 
929 	last_addr = efi_loadaddr(LOAD_MEM, &size, mfp->f_addr + mfp->f_size);
930 	mbi = (multiboot2_info_header_t *)last_addr;
931 	if (mbi == NULL) {
932 		error = ENOMEM;
933 		goto error;
934 	}
935 	last_addr = (vm_offset_t)mbi->mbi_tags;
936 #else
937 	/* Start info block from the new page. */
938 	last_addr = i386_loadaddr(LOAD_MEM, &size, mfp->f_addr + mfp->f_size);
939 
940 	/* Do we have space for multiboot info? */
941 	if (last_addr + size >= memtop_copyin) {
942 		error = ENOMEM;
943 		goto error;
944 	}
945 
946 	mbi = (multiboot2_info_header_t *)PTOV(last_addr);
947 	last_addr = (vm_offset_t)mbi->mbi_tags;
948 #endif	/* EFI */
949 
950 	{
951 		multiboot_tag_string_t *tag;
952 		i = sizeof (multiboot_tag_string_t) + strlen(cmdline) + 1;
953 		tag = (multiboot_tag_string_t *)mb_malloc(i);
954 
955 		tag->mb_type = MULTIBOOT_TAG_TYPE_CMDLINE;
956 		tag->mb_size = i;
957 		memcpy(tag->mb_string, cmdline, strlen(cmdline) + 1);
958 		free(cmdline);
959 		cmdline = NULL;
960 	}
961 
962 	{
963 		multiboot_tag_string_t *tag;
964 		i = sizeof (multiboot_tag_string_t) + strlen(bootprog_info) + 1;
965 		tag = (multiboot_tag_string_t *)mb_malloc(i);
966 
967 		tag->mb_type = MULTIBOOT_TAG_TYPE_BOOT_LOADER_NAME;
968 		tag->mb_size = i;
969 		memcpy(tag->mb_string, bootprog_info,
970 		    strlen(bootprog_info) + 1);
971 	}
972 
973 #if !defined(EFI)
974 	/* Only set in case of BIOS. */
975 	{
976 		multiboot_tag_basic_meminfo_t *tag;
977 		tag = (multiboot_tag_basic_meminfo_t *)
978 		    mb_malloc(sizeof (*tag));
979 
980 		tag->mb_type = MULTIBOOT_TAG_TYPE_BASIC_MEMINFO;
981 		tag->mb_size = sizeof (*tag);
982 		tag->mb_mem_lower = bios_basemem / 1024;
983 		tag->mb_mem_upper = bios_extmem / 1024;
984 	}
985 #endif
986 
987 	num = 0;
988 	for (mfp = fp->f_next; mfp != NULL; mfp = mfp->f_next) {
989 		num++;
990 		if (mfp->f_type != NULL && strcmp(mfp->f_type, "rootfs") == 0)
991 			rootfs++;
992 	}
993 
994 	if (num == 0 || rootfs == 0) {
995 		/* We need at least one module - rootfs. */
996 		printf("No rootfs module provided, aborting\n");
997 		error = EINVAL;
998 		goto error;
999 	}
1000 
1001 	/*
1002 	 * Set the stage for physical memory layout:
1003 	 * - We have kernel at load_addr.
1004 	 * - Modules are aligned to page boundary.
1005 	 * - MBI is aligned to page boundary.
1006 	 * - Set the tmp to point to physical address of the first module.
1007 	 * - tmp != mfp->f_addr only in case of EFI.
1008 	 */
1009 #if defined(EFI)
1010 	tmp = roundup2(load_addr + fp->f_size + 1, MULTIBOOT_MOD_ALIGN);
1011 	module = (multiboot_tag_module_t *)last_addr;
1012 #endif
1013 
1014 	for (mfp = fp->f_next; mfp != NULL; mfp = mfp->f_next) {
1015 		multiboot_tag_module_t *tag;
1016 
1017 		num = strlen(mfp->f_name) + 1;
1018 		num += strlen(mfp->f_type) + 5 + 1;
1019 		if (mfp->f_args != NULL) {
1020 			num += strlen(mfp->f_args) + 1;
1021 		}
1022 		cmdline = malloc(num);
1023 		if (cmdline == NULL) {
1024 			error = ENOMEM;
1025 			goto error;
1026 		}
1027 
1028 		if (mfp->f_args != NULL)
1029 			snprintf(cmdline, num, "%s type=%s %s",
1030 			    mfp->f_name, mfp->f_type, mfp->f_args);
1031 		else
1032 			snprintf(cmdline, num, "%s type=%s",
1033 			    mfp->f_name, mfp->f_type);
1034 
1035 		tag = (multiboot_tag_module_t *)mb_malloc(sizeof (*tag) + num);
1036 
1037 		tag->mb_type = MULTIBOOT_TAG_TYPE_MODULE;
1038 		tag->mb_size = sizeof (*tag) + num;
1039 #if defined(EFI)
1040 		/*
1041 		 * We can assign module addresses only after BS have been
1042 		 * switched off.
1043 		 */
1044 		tag->mb_mod_start = 0;
1045 		tag->mb_mod_end = mfp->f_size;
1046 #else
1047 		tag->mb_mod_start = mfp->f_addr;
1048 		tag->mb_mod_end = mfp->f_addr + mfp->f_size;
1049 #endif
1050 		memcpy(tag->mb_cmdline, cmdline, num);
1051 		free(cmdline);
1052 		cmdline = NULL;
1053 	}
1054 
1055 	md = file_findmetadata(fp, MODINFOMD_SMAP);
1056 	if (md == NULL) {
1057 		printf("no memory smap\n");
1058 		error = EINVAL;
1059 		goto error;
1060 	}
1061 
1062 	smap = (struct bios_smap *)md->md_data;
1063 	num = md->md_size / sizeof (struct bios_smap); /* number of entries */
1064 
1065 	{
1066 		multiboot_tag_mmap_t *tag;
1067 		multiboot_mmap_entry_t *mmap_entry;
1068 
1069 		tag = (multiboot_tag_mmap_t *)
1070 		    mb_malloc(sizeof (*tag) +
1071 		    num * sizeof (multiboot_mmap_entry_t));
1072 
1073 		tag->mb_type = MULTIBOOT_TAG_TYPE_MMAP;
1074 		tag->mb_size = sizeof (*tag) +
1075 		    num * sizeof (multiboot_mmap_entry_t);
1076 		tag->mb_entry_size = sizeof (multiboot_mmap_entry_t);
1077 		tag->mb_entry_version = 0;
1078 		mmap_entry = (multiboot_mmap_entry_t *)tag->mb_entries;
1079 
1080 		for (i = 0; i < num; i++) {
1081 			mmap_entry[i].mmap_addr = smap[i].base;
1082 			mmap_entry[i].mmap_len = smap[i].length;
1083 			mmap_entry[i].mmap_type = smap[i].type;
1084 			mmap_entry[i].mmap_reserved = 0;
1085 		}
1086 	}
1087 
1088 	if (bootp_response != NULL) {
1089 		multiboot_tag_network_t *tag;
1090 		tag = (multiboot_tag_network_t *)
1091 		    mb_malloc(sizeof (*tag) + bootp_response_size);
1092 
1093 		tag->mb_type = MULTIBOOT_TAG_TYPE_NETWORK;
1094 		tag->mb_size = sizeof (*tag) + bootp_response_size;
1095 		memcpy(tag->mb_dhcpack, bootp_response, bootp_response_size);
1096 	}
1097 
1098 #if !defined(EFI)
1099 	multiboot_tag_vbe_t *tag;
1100 	extern multiboot_tag_vbe_t vbestate;
1101 
1102 	if (VBE_VALID_MODE(vbestate.vbe_mode)) {
1103 		tag = (multiboot_tag_vbe_t *)mb_malloc(sizeof (*tag));
1104 		memcpy(tag, &vbestate, sizeof (*tag));
1105 		tag->mb_type = MULTIBOOT_TAG_TYPE_VBE;
1106 		tag->mb_size = sizeof (*tag);
1107 	}
1108 #endif
1109 
1110 	if (rsdp != NULL) {
1111 		multiboot_tag_new_acpi_t *ntag;
1112 		multiboot_tag_old_acpi_t *otag;
1113 		uint32_t tsize;
1114 
1115 		if (rsdp->Revision == 0) {
1116 			tsize = sizeof (*otag) + sizeof (ACPI_RSDP_COMMON);
1117 			otag = (multiboot_tag_old_acpi_t *)mb_malloc(tsize);
1118 			otag->mb_type = MULTIBOOT_TAG_TYPE_ACPI_OLD;
1119 			otag->mb_size = tsize;
1120 			memcpy(otag->mb_rsdp, rsdp, sizeof (ACPI_RSDP_COMMON));
1121 		} else {
1122 			tsize = sizeof (*ntag) + rsdp->Length;
1123 			ntag = (multiboot_tag_new_acpi_t *)mb_malloc(tsize);
1124 			ntag->mb_type = MULTIBOOT_TAG_TYPE_ACPI_NEW;
1125 			ntag->mb_size = tsize;
1126 			memcpy(ntag->mb_rsdp, rsdp, rsdp->Length);
1127 		}
1128 	}
1129 
1130 #if defined(EFI)
1131 #ifdef  __LP64__
1132 	{
1133 		multiboot_tag_efi64_t *tag;
1134 		tag = (multiboot_tag_efi64_t *)
1135 		    mb_malloc(sizeof (*tag));
1136 
1137 		tag->mb_type = MULTIBOOT_TAG_TYPE_EFI64;
1138 		tag->mb_size = sizeof (*tag);
1139 		tag->mb_pointer = (uint64_t)(uintptr_t)ST;
1140 	}
1141 #else
1142 	{
1143 		multiboot_tag_efi32_t *tag;
1144 		tag = (multiboot_tag_efi32_t *)
1145 		    mb_malloc(sizeof (*tag));
1146 
1147 		tag->mb_type = MULTIBOOT_TAG_TYPE_EFI32;
1148 		tag->mb_size = sizeof (*tag);
1149 		tag->mb_pointer = (uint32_t)ST;
1150 	}
1151 #endif /* __LP64__ */
1152 #endif /* EFI */
1153 
1154 	if (have_framebuffer == true) {
1155 		multiboot_tag_framebuffer_t *tag;
1156 		extern multiboot_tag_framebuffer_t gfx_fb;
1157 #if defined(EFI)
1158 
1159 		tag = (multiboot_tag_framebuffer_t *)mb_malloc(sizeof (*tag));
1160 		memcpy(tag, &gfx_fb, sizeof (*tag));
1161 		tag->framebuffer_common.mb_type =
1162 		    MULTIBOOT_TAG_TYPE_FRAMEBUFFER;
1163 		tag->framebuffer_common.mb_size = sizeof (*tag);
1164 #else
1165 		extern multiboot_color_t *cmap;
1166 		uint32_t size;
1167 
1168 		if (gfx_fb.framebuffer_common.framebuffer_type ==
1169 		    MULTIBOOT_FRAMEBUFFER_TYPE_INDEXED) {
1170 			uint16_t nc;
1171 			nc = gfx_fb.u.fb1.framebuffer_palette_num_colors;
1172 			size = sizeof (struct multiboot_tag_framebuffer_common)
1173 			    + sizeof (nc)
1174 			    + nc * sizeof (multiboot_color_t);
1175 		} else {
1176 			size = sizeof (gfx_fb);
1177 		}
1178 
1179 		tag = (multiboot_tag_framebuffer_t *)mb_malloc(size);
1180 		memcpy(tag, &gfx_fb, sizeof (*tag));
1181 
1182 		tag->framebuffer_common.mb_type =
1183 		    MULTIBOOT_TAG_TYPE_FRAMEBUFFER;
1184 		tag->framebuffer_common.mb_size = size;
1185 
1186 		if (gfx_fb.framebuffer_common.framebuffer_type ==
1187 		    MULTIBOOT_FRAMEBUFFER_TYPE_INDEXED) {
1188 			gfx_fb.u.fb1.framebuffer_palette_num_colors = CMAP_SIZE;
1189 
1190 			memcpy(tag->u.fb1.framebuffer_palette, cmap,
1191 			    sizeof (multiboot_color_t) * CMAP_SIZE);
1192 		}
1193 #endif /* EFI */
1194 	}
1195 
1196 #if defined(EFI)
1197 	/* Leave EFI memmap last as we will also switch off the BS. */
1198 	{
1199 		multiboot_tag_efi_mmap_t *tag;
1200 		UINTN key;
1201 		EFI_STATUS status;
1202 
1203 		tag = (multiboot_tag_efi_mmap_t *)
1204 		    mb_malloc(sizeof (*tag));
1205 
1206 		map_size = 0;
1207 		status = BS->GetMemoryMap(&map_size,
1208 		    (EFI_MEMORY_DESCRIPTOR *)tag->mb_efi_mmap, &key,
1209 		    &desc_size, &tag->mb_descr_vers);
1210 		if (status != EFI_BUFFER_TOO_SMALL) {
1211 			error = EINVAL;
1212 			goto error;
1213 		}
1214 		map_size = roundup2(map_size, EFI_PAGE_SIZE);
1215 
1216 		i = 2;	/* Attempts to ExitBootServices() */
1217 		while (map_size <= efi_map_size && i > 0) {
1218 			status = BS->GetMemoryMap(&map_size,
1219 			    (EFI_MEMORY_DESCRIPTOR *)tag->mb_efi_mmap, &key,
1220 			    &desc_size, &tag->mb_descr_vers);
1221 			if (status == EFI_BUFFER_TOO_SMALL) {
1222 				/* Still too small? */
1223 				map_size += EFI_PAGE_SIZE;
1224 				continue;
1225 			}
1226 			if (EFI_ERROR(status)) {
1227 				error = EINVAL;
1228 				goto error;
1229 			}
1230 
1231 			if (keep_bs != 0)
1232 				break;
1233 
1234 			status = BS->ExitBootServices(IH, key);
1235 			if (status == EFI_SUCCESS) {
1236 				has_boot_services = false;
1237 				break;
1238 			}
1239 			i--;
1240 		}
1241 		if (status != EFI_SUCCESS) {
1242 			error = EINVAL;
1243 			goto error;
1244 		}
1245 
1246 		tag->mb_type = MULTIBOOT_TAG_TYPE_EFI_MMAP;
1247 		tag->mb_size = sizeof (*tag) + map_size;
1248 		tag->mb_descr_size = (uint32_t)desc_size;
1249 
1250 		map = (EFI_MEMORY_DESCRIPTOR *)tag->mb_efi_mmap;
1251 
1252 		last_addr += map_size;
1253 		last_addr = roundup2(last_addr, MULTIBOOT_TAG_ALIGN);
1254 	}
1255 #endif /* EFI */
1256 
1257 	/*
1258 	 * MB tag list end marker.
1259 	 */
1260 	{
1261 		multiboot_tag_t *tag = (multiboot_tag_t *)
1262 		    mb_malloc(sizeof (*tag));
1263 		tag->mb_type = MULTIBOOT_TAG_TYPE_END;
1264 		tag->mb_size = sizeof (*tag);
1265 	}
1266 
1267 	mbi->mbi_total_size = last_addr - (vm_offset_t)mbi;
1268 	mbi->mbi_reserved = 0;
1269 
1270 #if defined(EFI)
1271 	/*
1272 	 * At this point we have load_addr pointing to kernel load
1273 	 * address, module list in MBI having physical addresses,
1274 	 * module list in fp having logical addresses and tmp pointing to
1275 	 * physical address for MBI.
1276 	 * Now we must move all pieces to place and start the kernel.
1277 	 */
1278 	head = &relocator->rel_chunk_head;
1279 	STAILQ_INIT(head);
1280 
1281 	i = 0;
1282 	chunk = &relocator->rel_chunklist[i++];
1283 	chunk->chunk_vaddr = fp->f_addr;
1284 	chunk->chunk_paddr = load_addr;
1285 	chunk->chunk_size = fp->f_size;
1286 
1287 	STAILQ_INSERT_TAIL(head, chunk, chunk_next);
1288 
1289 	mp = module;
1290 	for (mfp = fp->f_next; mfp != NULL; mfp = mfp->f_next) {
1291 		chunk = &relocator->rel_chunklist[i++];
1292 		chunk->chunk_vaddr = mfp->f_addr;
1293 
1294 		/*
1295 		 * fix the mb_mod_start and mb_mod_end.
1296 		 */
1297 		mp->mb_mod_start = efi_physaddr(module, tmp, map,
1298 		    map_size / desc_size, desc_size, mfp->f_addr,
1299 		    mp->mb_mod_end);
1300 		if (mp->mb_mod_start == 0)
1301 			panic("Could not find memory for module");
1302 
1303 		mp->mb_mod_end += mp->mb_mod_start;
1304 		chunk->chunk_paddr = mp->mb_mod_start;
1305 		chunk->chunk_size = mfp->f_size;
1306 		STAILQ_INSERT_TAIL(head, chunk, chunk_next);
1307 
1308 		mp = (multiboot_tag_module_t *)
1309 		    roundup2((uintptr_t)mp + mp->mb_size,
1310 		    MULTIBOOT_TAG_ALIGN);
1311 	}
1312 	chunk = &relocator->rel_chunklist[i++];
1313 	chunk->chunk_vaddr = (EFI_VIRTUAL_ADDRESS)(uintptr_t)mbi;
1314 	chunk->chunk_paddr = efi_physaddr(module, tmp, map,
1315 	    map_size / desc_size, desc_size, (uintptr_t)mbi,
1316 	    mbi->mbi_total_size);
1317 	chunk->chunk_size = mbi->mbi_total_size;
1318 	STAILQ_INSERT_TAIL(head, chunk, chunk_next);
1319 
1320 	trampoline = (void *)(uintptr_t)relocator + EFI_PAGE_SIZE;
1321 	memmove(trampoline, multiboot_tramp, EFI_PAGE_SIZE);
1322 
1323 	relocator->rel_copy = (uintptr_t)trampoline + EFI_PAGE_SIZE;
1324 	memmove((void *)relocator->rel_copy, efi_copy_finish, EFI_PAGE_SIZE);
1325 
1326 	relocator->rel_memmove = (uintptr_t)relocator->rel_copy + EFI_PAGE_SIZE;
1327 	memmove((void *)relocator->rel_memmove, memmove, EFI_PAGE_SIZE);
1328 	relocator->rel_stack = relocator->rel_memmove + EFI_PAGE_SIZE - 8;
1329 
1330 	trampoline(MULTIBOOT2_BOOTLOADER_MAGIC, relocator, entry_addr);
1331 #else
1332 	dev_cleanup();
1333 	__exec((void *)VTOP(multiboot_tramp), MULTIBOOT2_BOOTLOADER_MAGIC,
1334 	    (void *)entry_addr, (void *)VTOP(mbi));
1335 #endif /* EFI */
1336 	panic("exec returned");
1337 
1338 error:
1339 	free(cmdline);
1340 
1341 #if defined(EFI)
1342 	free(relocator);
1343 
1344 	if (mbi != NULL)
1345 		efi_free_loadaddr((vm_offset_t)mbi, EFI_SIZE_TO_PAGES(size));
1346 #endif
1347 
1348 	return (error);
1349 }
1350