0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018
0019
0020
0021
0022
0023
0024
0025
0026
0027
0028
0029
0030
0031
0032
0033
0034
0035
0036
0037
0038
0039
0040
0041
0042
0043
0044
0045
0046
0047
0048
0049
0050
0051
0052
0053
0054
0055
0056
0057
0058
0059
0060
0061
0062
0063
0064
0065
0066
0067
0068
0069
0070
0071
0072
0073
0074
0075 #include <stdio.h>
0076 #include <stdlib.h>
0077 #include <errno.h>
0078 #include <fcntl.h>
0079 #include <limits.h>
0080 #include <poll.h>
0081 #include <pthread.h>
0082 #include <string.h>
0083 #include <sys/eventfd.h>
0084 #include <sys/ioctl.h>
0085 #include <sys/mman.h>
0086 #include <sys/socket.h>
0087 #include <sys/stat.h>
0088 #include <sys/types.h>
0089 #include <unistd.h>
0090
0091 #include <linux/mman.h>
0092 #include <linux/nitro_enclaves.h>
0093 #include <linux/vm_sockets.h>
0094
0095
0096
0097
0098 #define NE_DEV_NAME "/dev/nitro_enclaves"
0099
0100
0101
0102
0103 #define NE_POLL_WAIT_TIME (60)
0104
0105
0106
0107 #define NE_POLL_WAIT_TIME_MS (NE_POLL_WAIT_TIME * 1000)
0108
0109
0110
0111
0112 #define NE_SLEEP_TIME (300)
0113
0114
0115
0116
0117 #define NE_DEFAULT_NR_VCPUS (2)
0118
0119
0120
0121
0122 #define NE_MIN_MEM_REGION_SIZE (2 * 1024 * 1024)
0123
0124
0125
0126
0127
0128 #define NE_DEFAULT_NR_MEM_REGIONS (256)
0129
0130
0131
0132
0133 #define NE_IMAGE_LOAD_HEARTBEAT_CID (3)
0134
0135
0136
0137 #define NE_IMAGE_LOAD_HEARTBEAT_PORT (9000)
0138
0139
0140
0141 #define NE_IMAGE_LOAD_HEARTBEAT_VALUE (0xb7)
0142
0143
0144
0145
0146
0147
0148 struct ne_user_mem_region {
0149 void *userspace_addr;
0150 size_t memory_size;
0151 };
0152
0153
0154
0155
0156
0157
0158
0159
0160
0161
0162
0163
0164 static int ne_create_vm(int ne_dev_fd, unsigned long *slot_uid, int *enclave_fd)
0165 {
0166 int rc = -EINVAL;
0167 *enclave_fd = ioctl(ne_dev_fd, NE_CREATE_VM, slot_uid);
0168
0169 if (*enclave_fd < 0) {
0170 rc = *enclave_fd;
0171 switch (errno) {
0172 case NE_ERR_NO_CPUS_AVAIL_IN_POOL: {
0173 printf("Error in create VM, no CPUs available in the NE CPU pool\n");
0174
0175 break;
0176 }
0177
0178 default:
0179 printf("Error in create VM [%m]\n");
0180 }
0181
0182 return rc;
0183 }
0184
0185 return 0;
0186 }
0187
0188
0189
0190
0191
0192
0193
0194
0195
0196 void *ne_poll_enclave_fd(void *data)
0197 {
0198 int enclave_fd = *(int *)data;
0199 struct pollfd fds[1] = {};
0200 int i = 0;
0201 int rc = -EINVAL;
0202
0203 printf("Running from poll thread, enclave fd %d\n", enclave_fd);
0204
0205 fds[0].fd = enclave_fd;
0206 fds[0].events = POLLIN | POLLERR | POLLHUP;
0207
0208
0209 while (1) {
0210 printf("[iter %d] Polling ...\n", i);
0211
0212 rc = poll(fds, 1, NE_POLL_WAIT_TIME_MS);
0213 if (rc < 0) {
0214 printf("Error in poll [%m]\n");
0215
0216 return NULL;
0217 }
0218
0219 i++;
0220
0221 if (!rc) {
0222 printf("Poll: %d seconds elapsed\n",
0223 i * NE_POLL_WAIT_TIME);
0224
0225 continue;
0226 }
0227
0228 printf("Poll received value 0x%x\n", fds[0].revents);
0229
0230 if (fds[0].revents & POLLHUP) {
0231 printf("Received POLLHUP\n");
0232
0233 return NULL;
0234 }
0235
0236 if (fds[0].revents & POLLNVAL) {
0237 printf("Received POLLNVAL\n");
0238
0239 return NULL;
0240 }
0241 }
0242
0243 return NULL;
0244 }
0245
0246
0247
0248
0249
0250
0251
0252
0253
0254
0255 static int ne_alloc_user_mem_region(struct ne_user_mem_region *ne_user_mem_region)
0256 {
0257
0258
0259
0260
0261 ne_user_mem_region->userspace_addr = mmap(NULL, ne_user_mem_region->memory_size,
0262 PROT_READ | PROT_WRITE,
0263 MAP_PRIVATE | MAP_ANONYMOUS |
0264 MAP_HUGETLB | MAP_HUGE_2MB, -1, 0);
0265 if (ne_user_mem_region->userspace_addr == MAP_FAILED) {
0266 printf("Error in mmap memory [%m]\n");
0267
0268 return -1;
0269 }
0270
0271 return 0;
0272 }
0273
0274
0275
0276
0277
0278
0279
0280
0281
0282
0283
0284
0285 static int ne_load_enclave_image(int enclave_fd, struct ne_user_mem_region ne_user_mem_regions[],
0286 char *enclave_image_path)
0287 {
0288 unsigned char *enclave_image = NULL;
0289 int enclave_image_fd = -1;
0290 size_t enclave_image_size = 0;
0291 size_t enclave_memory_size = 0;
0292 unsigned long i = 0;
0293 size_t image_written_bytes = 0;
0294 struct ne_image_load_info image_load_info = {
0295 .flags = NE_EIF_IMAGE,
0296 };
0297 struct stat image_stat_buf = {};
0298 int rc = -EINVAL;
0299 size_t temp_image_offset = 0;
0300
0301 for (i = 0; i < NE_DEFAULT_NR_MEM_REGIONS; i++)
0302 enclave_memory_size += ne_user_mem_regions[i].memory_size;
0303
0304 rc = stat(enclave_image_path, &image_stat_buf);
0305 if (rc < 0) {
0306 printf("Error in get image stat info [%m]\n");
0307
0308 return rc;
0309 }
0310
0311 enclave_image_size = image_stat_buf.st_size;
0312
0313 if (enclave_memory_size < enclave_image_size) {
0314 printf("The enclave memory is smaller than the enclave image size\n");
0315
0316 return -ENOMEM;
0317 }
0318
0319 rc = ioctl(enclave_fd, NE_GET_IMAGE_LOAD_INFO, &image_load_info);
0320 if (rc < 0) {
0321 switch (errno) {
0322 case NE_ERR_NOT_IN_INIT_STATE: {
0323 printf("Error in get image load info, enclave not in init state\n");
0324
0325 break;
0326 }
0327
0328 case NE_ERR_INVALID_FLAG_VALUE: {
0329 printf("Error in get image load info, provided invalid flag\n");
0330
0331 break;
0332 }
0333
0334 default:
0335 printf("Error in get image load info [%m]\n");
0336 }
0337
0338 return rc;
0339 }
0340
0341 printf("Enclave image offset in enclave memory is %lld\n",
0342 image_load_info.memory_offset);
0343
0344 enclave_image_fd = open(enclave_image_path, O_RDONLY);
0345 if (enclave_image_fd < 0) {
0346 printf("Error in open enclave image file [%m]\n");
0347
0348 return enclave_image_fd;
0349 }
0350
0351 enclave_image = mmap(NULL, enclave_image_size, PROT_READ,
0352 MAP_PRIVATE, enclave_image_fd, 0);
0353 if (enclave_image == MAP_FAILED) {
0354 printf("Error in mmap enclave image [%m]\n");
0355
0356 return -1;
0357 }
0358
0359 temp_image_offset = image_load_info.memory_offset;
0360
0361 for (i = 0; i < NE_DEFAULT_NR_MEM_REGIONS; i++) {
0362 size_t bytes_to_write = 0;
0363 size_t memory_offset = 0;
0364 size_t memory_size = ne_user_mem_regions[i].memory_size;
0365 size_t remaining_bytes = 0;
0366 void *userspace_addr = ne_user_mem_regions[i].userspace_addr;
0367
0368 if (temp_image_offset >= memory_size) {
0369 temp_image_offset -= memory_size;
0370
0371 continue;
0372 } else if (temp_image_offset != 0) {
0373 memory_offset = temp_image_offset;
0374 memory_size -= temp_image_offset;
0375 temp_image_offset = 0;
0376 }
0377
0378 remaining_bytes = enclave_image_size - image_written_bytes;
0379 bytes_to_write = memory_size < remaining_bytes ?
0380 memory_size : remaining_bytes;
0381
0382 memcpy(userspace_addr + memory_offset,
0383 enclave_image + image_written_bytes, bytes_to_write);
0384
0385 image_written_bytes += bytes_to_write;
0386
0387 if (image_written_bytes == enclave_image_size)
0388 break;
0389 }
0390
0391 munmap(enclave_image, enclave_image_size);
0392
0393 close(enclave_image_fd);
0394
0395 return 0;
0396 }
0397
0398
0399
0400
0401
0402
0403
0404
0405
0406
0407
0408 static int ne_set_user_mem_region(int enclave_fd, struct ne_user_mem_region ne_user_mem_region)
0409 {
0410 struct ne_user_memory_region mem_region = {
0411 .flags = NE_DEFAULT_MEMORY_REGION,
0412 .memory_size = ne_user_mem_region.memory_size,
0413 .userspace_addr = (__u64)ne_user_mem_region.userspace_addr,
0414 };
0415 int rc = -EINVAL;
0416
0417 rc = ioctl(enclave_fd, NE_SET_USER_MEMORY_REGION, &mem_region);
0418 if (rc < 0) {
0419 switch (errno) {
0420 case NE_ERR_NOT_IN_INIT_STATE: {
0421 printf("Error in set user memory region, enclave not in init state\n");
0422
0423 break;
0424 }
0425
0426 case NE_ERR_INVALID_MEM_REGION_SIZE: {
0427 printf("Error in set user memory region, mem size not multiple of 2 MiB\n");
0428
0429 break;
0430 }
0431
0432 case NE_ERR_INVALID_MEM_REGION_ADDR: {
0433 printf("Error in set user memory region, invalid user space address\n");
0434
0435 break;
0436 }
0437
0438 case NE_ERR_UNALIGNED_MEM_REGION_ADDR: {
0439 printf("Error in set user memory region, unaligned user space address\n");
0440
0441 break;
0442 }
0443
0444 case NE_ERR_MEM_REGION_ALREADY_USED: {
0445 printf("Error in set user memory region, memory region already used\n");
0446
0447 break;
0448 }
0449
0450 case NE_ERR_MEM_NOT_HUGE_PAGE: {
0451 printf("Error in set user memory region, not backed by huge pages\n");
0452
0453 break;
0454 }
0455
0456 case NE_ERR_MEM_DIFFERENT_NUMA_NODE: {
0457 printf("Error in set user memory region, different NUMA node than CPUs\n");
0458
0459 break;
0460 }
0461
0462 case NE_ERR_MEM_MAX_REGIONS: {
0463 printf("Error in set user memory region, max memory regions reached\n");
0464
0465 break;
0466 }
0467
0468 case NE_ERR_INVALID_PAGE_SIZE: {
0469 printf("Error in set user memory region, has page not multiple of 2 MiB\n");
0470
0471 break;
0472 }
0473
0474 case NE_ERR_INVALID_FLAG_VALUE: {
0475 printf("Error in set user memory region, provided invalid flag\n");
0476
0477 break;
0478 }
0479
0480 default:
0481 printf("Error in set user memory region [%m]\n");
0482 }
0483
0484 return rc;
0485 }
0486
0487 return 0;
0488 }
0489
0490
0491
0492
0493
0494
0495
0496
0497 static void ne_free_mem_regions(struct ne_user_mem_region ne_user_mem_regions[])
0498 {
0499 unsigned int i = 0;
0500
0501 for (i = 0; i < NE_DEFAULT_NR_MEM_REGIONS; i++)
0502 munmap(ne_user_mem_regions[i].userspace_addr,
0503 ne_user_mem_regions[i].memory_size);
0504 }
0505
0506
0507
0508
0509
0510
0511
0512
0513
0514
0515
0516
0517 static int ne_add_vcpu(int enclave_fd, unsigned int *vcpu_id)
0518 {
0519 int rc = -EINVAL;
0520
0521 rc = ioctl(enclave_fd, NE_ADD_VCPU, vcpu_id);
0522 if (rc < 0) {
0523 switch (errno) {
0524 case NE_ERR_NO_CPUS_AVAIL_IN_POOL: {
0525 printf("Error in add vcpu, no CPUs available in the NE CPU pool\n");
0526
0527 break;
0528 }
0529
0530 case NE_ERR_VCPU_ALREADY_USED: {
0531 printf("Error in add vcpu, the provided vCPU is already used\n");
0532
0533 break;
0534 }
0535
0536 case NE_ERR_VCPU_NOT_IN_CPU_POOL: {
0537 printf("Error in add vcpu, the provided vCPU is not in the NE CPU pool\n");
0538
0539 break;
0540 }
0541
0542 case NE_ERR_VCPU_INVALID_CPU_CORE: {
0543 printf("Error in add vcpu, the core id of the provided vCPU is invalid\n");
0544
0545 break;
0546 }
0547
0548 case NE_ERR_NOT_IN_INIT_STATE: {
0549 printf("Error in add vcpu, enclave not in init state\n");
0550
0551 break;
0552 }
0553
0554 case NE_ERR_INVALID_VCPU: {
0555 printf("Error in add vcpu, the provided vCPU is out of avail CPUs range\n");
0556
0557 break;
0558 }
0559
0560 default:
0561 printf("Error in add vcpu [%m]\n");
0562 }
0563
0564 return rc;
0565 }
0566
0567 return 0;
0568 }
0569
0570
0571
0572
0573
0574
0575
0576
0577
0578
0579
0580 static int ne_start_enclave(int enclave_fd, struct ne_enclave_start_info *enclave_start_info)
0581 {
0582 int rc = -EINVAL;
0583
0584 rc = ioctl(enclave_fd, NE_START_ENCLAVE, enclave_start_info);
0585 if (rc < 0) {
0586 switch (errno) {
0587 case NE_ERR_NOT_IN_INIT_STATE: {
0588 printf("Error in start enclave, enclave not in init state\n");
0589
0590 break;
0591 }
0592
0593 case NE_ERR_NO_MEM_REGIONS_ADDED: {
0594 printf("Error in start enclave, no memory regions have been added\n");
0595
0596 break;
0597 }
0598
0599 case NE_ERR_NO_VCPUS_ADDED: {
0600 printf("Error in start enclave, no vCPUs have been added\n");
0601
0602 break;
0603 }
0604
0605 case NE_ERR_FULL_CORES_NOT_USED: {
0606 printf("Error in start enclave, enclave has no full cores set\n");
0607
0608 break;
0609 }
0610
0611 case NE_ERR_ENCLAVE_MEM_MIN_SIZE: {
0612 printf("Error in start enclave, enclave memory is less than min size\n");
0613
0614 break;
0615 }
0616
0617 case NE_ERR_INVALID_FLAG_VALUE: {
0618 printf("Error in start enclave, provided invalid flag\n");
0619
0620 break;
0621 }
0622
0623 case NE_ERR_INVALID_ENCLAVE_CID: {
0624 printf("Error in start enclave, provided invalid enclave CID\n");
0625
0626 break;
0627 }
0628
0629 default:
0630 printf("Error in start enclave [%m]\n");
0631 }
0632
0633 return rc;
0634 }
0635
0636 return 0;
0637 }
0638
0639
0640
0641
0642
0643
0644
0645
0646
0647
0648
0649
0650 static int ne_start_enclave_check_booted(int enclave_fd)
0651 {
0652 struct sockaddr_vm client_vsock_addr = {};
0653 int client_vsock_fd = -1;
0654 socklen_t client_vsock_len = sizeof(client_vsock_addr);
0655 struct ne_enclave_start_info enclave_start_info = {};
0656 struct pollfd fds[1] = {};
0657 int rc = -EINVAL;
0658 unsigned char recv_buf = 0;
0659 struct sockaddr_vm server_vsock_addr = {
0660 .svm_family = AF_VSOCK,
0661 .svm_cid = NE_IMAGE_LOAD_HEARTBEAT_CID,
0662 .svm_port = NE_IMAGE_LOAD_HEARTBEAT_PORT,
0663 };
0664 int server_vsock_fd = -1;
0665
0666 server_vsock_fd = socket(AF_VSOCK, SOCK_STREAM, 0);
0667 if (server_vsock_fd < 0) {
0668 rc = server_vsock_fd;
0669
0670 printf("Error in socket [%m]\n");
0671
0672 return rc;
0673 }
0674
0675 rc = bind(server_vsock_fd, (struct sockaddr *)&server_vsock_addr,
0676 sizeof(server_vsock_addr));
0677 if (rc < 0) {
0678 printf("Error in bind [%m]\n");
0679
0680 goto out;
0681 }
0682
0683 rc = listen(server_vsock_fd, 1);
0684 if (rc < 0) {
0685 printf("Error in listen [%m]\n");
0686
0687 goto out;
0688 }
0689
0690 rc = ne_start_enclave(enclave_fd, &enclave_start_info);
0691 if (rc < 0)
0692 goto out;
0693
0694 printf("Enclave started, CID %llu\n", enclave_start_info.enclave_cid);
0695
0696 fds[0].fd = server_vsock_fd;
0697 fds[0].events = POLLIN;
0698
0699 rc = poll(fds, 1, NE_POLL_WAIT_TIME_MS);
0700 if (rc < 0) {
0701 printf("Error in poll [%m]\n");
0702
0703 goto out;
0704 }
0705
0706 if (!rc) {
0707 printf("Poll timeout, %d seconds elapsed\n", NE_POLL_WAIT_TIME);
0708
0709 rc = -ETIMEDOUT;
0710
0711 goto out;
0712 }
0713
0714 if ((fds[0].revents & POLLIN) == 0) {
0715 printf("Poll received value %d\n", fds[0].revents);
0716
0717 rc = -EINVAL;
0718
0719 goto out;
0720 }
0721
0722 rc = accept(server_vsock_fd, (struct sockaddr *)&client_vsock_addr,
0723 &client_vsock_len);
0724 if (rc < 0) {
0725 printf("Error in accept [%m]\n");
0726
0727 goto out;
0728 }
0729
0730 client_vsock_fd = rc;
0731
0732
0733
0734
0735
0736 rc = read(client_vsock_fd, &recv_buf, sizeof(recv_buf));
0737 if (rc < 0) {
0738 printf("Error in read [%m]\n");
0739
0740 goto out;
0741 }
0742
0743 if (rc != sizeof(recv_buf) || recv_buf != NE_IMAGE_LOAD_HEARTBEAT_VALUE) {
0744 printf("Read %d instead of %d\n", recv_buf,
0745 NE_IMAGE_LOAD_HEARTBEAT_VALUE);
0746
0747 goto out;
0748 }
0749
0750
0751 rc = write(client_vsock_fd, &recv_buf, sizeof(recv_buf));
0752 if (rc < 0) {
0753 printf("Error in write [%m]\n");
0754
0755 goto out;
0756 }
0757
0758 rc = 0;
0759
0760 out:
0761 close(server_vsock_fd);
0762
0763 return rc;
0764 }
0765
0766 int main(int argc, char *argv[])
0767 {
0768 int enclave_fd = -1;
0769 unsigned int i = 0;
0770 int ne_dev_fd = -1;
0771 struct ne_user_mem_region ne_user_mem_regions[NE_DEFAULT_NR_MEM_REGIONS] = {};
0772 unsigned int ne_vcpus[NE_DEFAULT_NR_VCPUS] = {};
0773 int rc = -EINVAL;
0774 pthread_t thread_id = 0;
0775 unsigned long slot_uid = 0;
0776
0777 if (argc != 2) {
0778 printf("Usage: %s <path_to_enclave_image>\n", argv[0]);
0779
0780 exit(EXIT_FAILURE);
0781 }
0782
0783 if (strlen(argv[1]) >= PATH_MAX) {
0784 printf("The size of the path to enclave image is higher than max path\n");
0785
0786 exit(EXIT_FAILURE);
0787 }
0788
0789 ne_dev_fd = open(NE_DEV_NAME, O_RDWR | O_CLOEXEC);
0790 if (ne_dev_fd < 0) {
0791 printf("Error in open NE device [%m]\n");
0792
0793 exit(EXIT_FAILURE);
0794 }
0795
0796 printf("Creating enclave slot ...\n");
0797
0798 rc = ne_create_vm(ne_dev_fd, &slot_uid, &enclave_fd);
0799
0800 close(ne_dev_fd);
0801
0802 if (rc < 0)
0803 exit(EXIT_FAILURE);
0804
0805 printf("Enclave fd %d\n", enclave_fd);
0806
0807 rc = pthread_create(&thread_id, NULL, ne_poll_enclave_fd, (void *)&enclave_fd);
0808 if (rc < 0) {
0809 printf("Error in thread create [%m]\n");
0810
0811 close(enclave_fd);
0812
0813 exit(EXIT_FAILURE);
0814 }
0815
0816 for (i = 0; i < NE_DEFAULT_NR_MEM_REGIONS; i++) {
0817 ne_user_mem_regions[i].memory_size = NE_MIN_MEM_REGION_SIZE;
0818
0819 rc = ne_alloc_user_mem_region(&ne_user_mem_regions[i]);
0820 if (rc < 0) {
0821 printf("Error in alloc userspace memory region, iter %d\n", i);
0822
0823 goto release_enclave_fd;
0824 }
0825 }
0826
0827 rc = ne_load_enclave_image(enclave_fd, ne_user_mem_regions, argv[1]);
0828 if (rc < 0)
0829 goto release_enclave_fd;
0830
0831 for (i = 0; i < NE_DEFAULT_NR_MEM_REGIONS; i++) {
0832 rc = ne_set_user_mem_region(enclave_fd, ne_user_mem_regions[i]);
0833 if (rc < 0) {
0834 printf("Error in set memory region, iter %d\n", i);
0835
0836 goto release_enclave_fd;
0837 }
0838 }
0839
0840 printf("Enclave memory regions were added\n");
0841
0842 for (i = 0; i < NE_DEFAULT_NR_VCPUS; i++) {
0843
0844
0845
0846
0847 ne_vcpus[i] = 0;
0848 rc = ne_add_vcpu(enclave_fd, &ne_vcpus[i]);
0849 if (rc < 0) {
0850 printf("Error in add vcpu, iter %d\n", i);
0851
0852 goto release_enclave_fd;
0853 }
0854
0855 printf("Added vCPU %d to the enclave\n", ne_vcpus[i]);
0856 }
0857
0858 printf("Enclave vCPUs were added\n");
0859
0860 rc = ne_start_enclave_check_booted(enclave_fd);
0861 if (rc < 0) {
0862 printf("Error in the enclave start / image loading heartbeat logic [rc=%d]\n", rc);
0863
0864 goto release_enclave_fd;
0865 }
0866
0867 printf("Entering sleep for %d seconds ...\n", NE_SLEEP_TIME);
0868
0869 sleep(NE_SLEEP_TIME);
0870
0871 close(enclave_fd);
0872
0873 ne_free_mem_regions(ne_user_mem_regions);
0874
0875 exit(EXIT_SUCCESS);
0876
0877 release_enclave_fd:
0878 close(enclave_fd);
0879 ne_free_mem_regions(ne_user_mem_regions);
0880
0881 exit(EXIT_FAILURE);
0882 }