/************************************************************************** * * Copyright (c) 2020-2023 by Guangzhou Pixtalks Information Technology Co., Ltd * * This software is copyrighted by and is the property of Guangzhou Pixtalks * Information Technology Co., Ltd. All rights are reserved by Guangzhou * Pixtalks Information Technology Co., Ltd. This software may only be used * in accordance with the corresponding license agreement. Any unauthorized * use, duplication, distribution, or disclosure of this software is * expressly forbidden. * * This Copyright notice MUST not be removed or modified without prior * written consent of Guangzhou Pixtalks Information Technology Co., Ltd. * * Guangzhou Pixtalks Information Technology Co., Ltd. reserves the right to * modify this software without notice. * **************************************************************************/ #include #include #include #include #include "p_code.h" #include "pix_algokit_public_type.h" #include "pix_facekit_api.h" #include "pix_palmkit_api.h" #include "vip_lite.h" #include "aw_util.h" #include "rt_pixtalks_face.h" /* register face img*/ #define NIR_IMG_PATH "/bin/640x480_nv21_vipp1_10.yuv" #define RGB_IMG_PATH "/bin/640x480_nv21_vipp0_10.yuv" /* model file */ #define DETECT_MODEL_FILE_PATH "/lib/face_hand_detect_v1_7_20230725.bin" #define FACE_MODEL_FILE_PATH "/lib/pix_face_v1_5_20230608.bin" #define FACE_LAZY_MODEL_FILE_PATH "/lib/pix_face_lazy_v1_0_20230515.bin" /* board_00 */ const unsigned char pix_license_auth_key[] = { 0x4, 0xea, 0x3e, 0x7b, 0xb1, 0x5e, 0x39, 0x96, 0x65, 0x30, 0xe6, 0xde, 0x95, 0xec, 0x5d, 0xa9, 0x66, 0xc9, 0x5e, 0xe9, 0x66, 0x7, 0xa4, 0x9c, 0x29, 0x93, 0xc5, 0x43, 0x18, 0x59, 0xf1, 0x65, 0x56, 0x1c, 0xcb, 0xc9, 0x6e, 0xfb, 0xd1, 0x35, 0x5b, 0xda, 0x68, 0xb2, 0x15, 0x5, 0xf9, 0x98, 0xcc, 0xe7, 0x38, 0xad, 0xa7, 0x96, 0xd8, 0x21, 0xf1, 0xa8, 0x2, 0x3b, 0x44, 0x5a, 0xa3, 0x72, 0x3a, 0xa8, 0x3a, 0x18, 0xeb, 0x6, 0x40, 0x5f, 0x2e, 0xa8, 0x65, 0xa7, 0xdd, 0x8c, 0xd9, 0x3d, 0xf5, 0x38, 0x1e, 0xb9, 0xdf, 0xa8, 0xf8, 0xff, 0x67, 0x87, 0xef, 0xf2, 0x17, 0x8b, 0x97, 0xf7, 0xef, 0x9a, 0x9f, 0xf4, 0xf6, 0xe0, 0x20, 0xb4, 0x78, 0xc8, 0xe9, 0x9f, 0xdd, 0xdd, 0x95, 0x39, 0x72, 0x9f, 0x7a, 0x68, 0x6d, 0x22, 0x66, 0x8d, 0x25, 0x28, 0x6, 0x1b, 0xdb, 0x2a, 0x11, 0x4a }; /* board_02 */ //const unsigned char pix_license_auth_key[] = { // 0x70, 0x58, 0x48, 0x88, 0x95, 0x1e, 0x38, 0xcc, 0xd7, 0xde, 0xce, 0x1f, 0x87, 0x6d, 0x8a, 0x17, 0x17, 0x14, 0x37, 0x19, 0x95, 0x7a, 0x7c, 0x5d, 0x38, 0x17, 0xb1, 0xe6, 0x67, 0x70, 0x71, 0x6, 0xba, 0xa7, 0xb, 0x2a, 0x51, 0x9, 0x45, 0xb9, 0x1a, 0x21, 0xd4, 0xf7, 0xb8, 0x18, 0x5c, 0x12, 0x27, 0x53, 0x45, 0x6c, 0xd2, 0xcb, 0xd4, 0xd4, 0x48, 0x5f, 0xf0, 0x0, 0x7f, 0xba, 0xe5, 0xec, 0xbb, 0x80, 0x6, 0x12, 0xfe, 0xea, 0xb2, 0xe0, 0xa0, 0xb9, 0x6b, 0x47, 0x38, 0xe2, 0x89, 0x8f, 0xad, 0x2c, 0x74, 0xb6, 0x34, 0xe7, 0x5f, 0x6, 0x1c, 0x9c, 0xc, 0xe0, 0x88, 0x6e, 0x2b, 0x60, 0x9b, 0x17, 0x8c, 0xe5, 0xbf, 0xeb, 0x83, 0x36, 0xef, 0x5d, 0xe6, 0xdc, 0xf6, 0xa5, 0x5d, 0x5c, 0xb4, 0x2e, 0x37, 0xa, 0x68, 0x33, 0x7c, 0x7a, 0xb8, 0xc9, 0x11, 0x4d, 0x65, 0x77, 0xf4, 0xc8 //}; /* board_01 */ //const unsigned char pix_license_auth_key[] = { // 0xad, 0x98, 0x51, 0xcf, 0x2f, 0xb1, 0x2b, 0xdb, 0xd2, 0x20, 0xa7, 0x4, 0x4b, 0x5d, 0x80, 0x19, 0x99, 0x82, 0x8c, 0x90, 0x7, 0x6e, 0x9a, 0xe8, 0x24, 0x7f, 0xf, 0xf6, 0x2e, 0x49, 0xdc, 0x90, 0x9c, 0x8d, 0xab, 0xeb, 0xd4, 0x1d, 0xe7, 0x72, 0x9b, 0x61, 0xc, 0xc1, 0xfd, 0xcb, 0x91, 0xd0, 0xe, 0x38, 0xa3, 0xba, 0xa3, 0x3, 0x50, 0xba, 0xb7, 0x3a, 0x2e, 0x8e, 0x38, 0x29, 0x2e, 0xc7, 0x72, 0xee, 0x8, 0xd9, 0x63, 0x15, 0x91, 0xfe, 0x2f, 0x2b, 0x4c, 0xc1, 0x1, 0x33, 0x7, 0x7a, 0x91, 0x9a, 0x16, 0x13, 0xc2, 0xed, 0x67, 0xe, 0xca, 0xf3, 0x7c, 0xc3, 0xcc, 0xc8, 0x40, 0xf8, 0x9b, 0x66, 0xfc, 0x6f, 0x34, 0xf0, 0x4a, 0x58, 0x22, 0xd2, 0x39, 0x20, 0xf1, 0x87, 0xd0, 0x39, 0xd3, 0xc9, 0x9d, 0xc2, 0xf1, 0x5b, 0x93, 0x36, 0x79, 0x2, 0x85, 0xb1, 0x7a, 0x22, 0x2a, 0x2e //}; // inner debug flag #ifndef PIX_BUILD_LEVEL #define PIX_BUILD_LEVEL -100 #define PIX_BUILD_LEVEL_INNER_ROC_TEST 5 #endif // PIX_BUILD_LEVEL #define NIR_IMG_WIDTH 480 #define NIR_IMG_HEIGHT 640 #define RGB_IMG_WIDTH 480 #define RGB_IMG_HEIGHT 640 #define PIX_HARDWARE_INFO_BYTES 32 #define CARE_INSIDE_IMAGE_PERSON_NUM 1 //只对画面中CARE_INSIDE_IMAGE_PERSON_NUM个人脸进行活体分析/识别特征抽取. 检测到多人脸时, 按像素面积排序后返回检测结果. #ifndef __PIX_ONE_FACE_INFO_S__ #define __PIX_ONE_FACE_INFO_S__ typedef struct pix_one_face_info_s{ char name[1024]; float box[5]; pix_face_attr_info_t attr; pix_fr_feature_info_t fr_feature; }pix_one_face_info_t; #endif //__PIX_ONE_FACE_INFO_S__ #ifndef __PIX_ONE_HAND_INFO_S__ #define __PIX_ONE_HAND_INFO_S__ typedef struct pix_one_hand_info_s { float box[5]; float landmarks[3 * 21]; int hand_quality; float liveness1_score; float liveness2_score; unsigned char pr_feature[PIX_PR_FEATURE_BYTES]; }pix_one_hand_info_t; #endif //__PIX_ONE_HAND_INFO_S__ #define BILLION 1000000000 static vip_uint64_t get_perf_count() { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (vip_uint64_t)((vip_uint64_t)ts.tv_nsec + (vip_uint64_t)ts.tv_sec * BILLION); } #include float get_current_timestamp_ms() { struct timeval tv; gettimeofday(&tv, NULL); return tv.tv_sec * 1000.0 + tv.tv_usec / 1000.0; } // the file size must large than disk mini unit or fail int read_file_to_malloc_heap(const char *file_path, unsigned char **content_ptr, int *file_size) { *content_ptr = NULL; FILE *file = fopen(file_path, "rb"); if (!file) { aw_loge("Fail to open file %s\n", file_path); return SDK_CODE_INVALID_ARG; } fseek(file, 0, SEEK_END); *file_size = ftell(file); fseek(file, 0, SEEK_SET); if (*file_size < 1) { aw_loge("Bad file %d\n", file_size); fclose(file); return SDK_CODE_INVALID_ARG; } *content_ptr = (unsigned char *)malloc(*file_size); if(! content_ptr) { aw_loge("Fail to malloc bytes %d\n", *file_size); return SDK_CODE_INNER_ERROR; } int read_size = fread(*content_ptr, *file_size, 1, file); if (read_size < 1) { aw_loge("Fail to read file with %d\n", read_size); if (*content_ptr) { free(*content_ptr); } fclose(file); return SDK_CODE_INVALID_ARG; } fclose(file); return SDK_CODE_OK; } int read_face_image_pair_to_malloc(const unsigned char *nir_img_path, const unsigned char *rgb_img_path, unsigned char **pair_img1_ptr, unsigned char **pair_img2_ptr) { const int nir_img_bytes = sizeof(unsigned char) * NIR_IMG_WIDTH * NIR_IMG_HEIGHT * 3 / 2; const int rgb_img_bytes = sizeof(unsigned char) * RGB_IMG_WIDTH * RGB_IMG_HEIGHT * 3 / 2; int read_img_bytes = 0; int ret = read_file_to_malloc_heap(nir_img_path, pair_img1_ptr, &read_img_bytes); if(ret != SDK_CODE_OK) { aw_loge("Fail to read %s\n", nir_img_path); return ret; } if(read_img_bytes != nir_img_bytes) { aw_loge("Bad nir img bytes %d(expect %d)\n", read_img_bytes, nir_img_bytes); return SDK_CODE_INVALID_ARG; } ret = read_file_to_malloc_heap(rgb_img_path, pair_img2_ptr, &read_img_bytes); if(ret != SDK_CODE_OK) { aw_loge("Fail to read %s\n", rgb_img_path); return ret; } if(read_img_bytes != nir_img_bytes) { aw_loge("Bad rgb img bytes %d(expect %d)\n", read_img_bytes, rgb_img_bytes); return SDK_CODE_INVALID_ARG; } return SDK_CODE_OK; } void dump_pix_coord_info(pix_coord_info_t *coord_info) { aw_logd("Box number %d\n", coord_info->box_number); for(int i = 0; i < coord_info->box_number; i++) { aw_logd("\nbox %d coord info ", i); for(int j = 0; j < API_PER_BOX_ELEMENTS - 1; j++) { printf("%f ", coord_info->box[i * API_PER_BOX_ELEMENTS + j]); } printf("\n"); } aw_logd("\n"); } static int get_face_info(pix_image_face_info_t *face_info, int is_register_flag, const unsigned char *pair_img1_ptr, unsigned char *pair_img2_ptr) { int ret = pix_set_normal_analyze_image(pair_img1_ptr, pair_img2_ptr); if(SDK_CODE_OK != ret) { aw_loge("Fail to pix_set_normal_analyze_image with %d\n", ret); return ret; } ret = pix_normal_fd(&face_info->coord); if(SDK_CODE_OK != ret) { aw_loge("Fail to pix_normal_fd with %d\n", ret); return ret; } // dump_pix_coord_info(&face_info->coord); // aw_logd("the face info!!!\r\n"); if(face_info->coord.box_number > CARE_INSIDE_IMAGE_PERSON_NUM) { aw_logd("Detect %d person, but only care %d(order by pixel area)\n", face_info->coord.box_number, CARE_INSIDE_IMAGE_PERSON_NUM); face_info->coord.box_number = CARE_INSIDE_IMAGE_PERSON_NUM; } for(int i = 0; i < face_info->coord.box_number; i++) { ret = pix_liveness_check(i, &face_info->liveness[i].score1); if(SDK_CODE_OK != ret) { printf("Fail to pix_liveness_check with %d\n", ret); return ret; } aw_logd("liveness score is %f \n", face_info->liveness[i].score1); // 注册阶段要求活体通过、没遮挡且姿态角符合预设值，才抽取识别特征入库 // 识别阶段可以只要活体通过就抽取识别特征与底库进行比对（跳过遮挡、姿态角判断） // 识别阶段活体没过，可以调用遮挡接口判断脸部是否有遮挡、调用姿态接口判断角度是否超规格，若超了提示“正视摄像头”等相应信息。 if(face_info->liveness[i].score1 < pix_get_liveness_threshold()) { aw_logd("Is not person\n"); // return } else { aw_logd("Is person\n"); } float mask_score = 0; ret = pix_with_mask_confidence(i, &mask_score); if(SDK_CODE_OK != ret) { aw_loge("Fail to pix_mask_check with %d\n", ret); return ret; } aw_logd("the mask score is %f\n", mask_score); pix_face_attr_info_t face_attr; ret = pix_get_face_attr_info(i, &face_attr); if(SDK_CODE_OK != ret) { aw_loge("Fail to pix_get_face_attr_info with %d\n", ret); return ret; } aw_logd("the face attr info %f %f %f %f\n", face_attr.distance, face_attr.left_right, face_attr.roll, face_attr.up_down); ret = pix_get_face_fr_feature(i, is_register_flag, &face_info->fr_feature[i]); if(SDK_CODE_OK != ret) { aw_loge("Fail to pix_get_face_fr_feature with %d\n", ret); return ret; } #if 1 aw_logd("the face feature is \n"); for(int j = 0; j < face_info->fr_feature->fea_length; j++) { printf("0x%02x, ", face_info->fr_feature[i].fea[j]); } aw_logd("\n"); #endif //aw_logd("pix_get_face_fr_feature end\n"); } return SDK_CODE_OK; } int rt_pixtalks_face_init(int npu_memory_bytes) { vip_status_e vip_init_ret = vip_init(npu_memory_bytes); if(vip_init_ret != VIP_SUCCESS){ aw_loge("Fail to vip_init with %d\n", vip_init_ret); return SDK_CODE_INNER_ERROR; } /* get hardware info */ unsigned char hardware_info[PIX_HARDWARE_INFO_BYTES] = {0}; int ret = pix_get_hardware_info(hardware_info, PIX_HARDWARE_INFO_BYTES); if(ret != SDK_CODE_OK){ aw_loge("Fail to get pix_get_hardware_info with %d\n", ret); return ret; } printf("Hardware info is:\n"); for (uint32_t i = 0; i < PIX_HARDWARE_INFO_BYTES; i++){ printf("0x%02x,", hardware_info[i]); } printf(".\n"); pix_board_product_id_e product_id = (pix_board_product_id_e)pix_product_id_hand_only_ir_rgb; pix_set_product_id(product_id); return 0; } int rt_pixtalks_face_create(struct rt_pixtalks_face_config_param* face_model_config) { int ret = -1; pix_auth_info_t pix_auth; pix_bool_t is_fd_algo_only = 0; pix_auth.pix_licence = pix_license_auth_key; pix_detect_target_type_e target_type = pix_detect_target_type_face; /* read face img to register face info */ /* TODO: register more faces */ unsigned char *nir_img_ptr = NULL; unsigned char *rgb_img_ptr = NULL; unsigned char nir_img_path[] = NIR_IMG_PATH; unsigned char rgb_img_path[] = RGB_IMG_PATH; ret = read_face_image_pair_to_malloc(nir_img_path, rgb_img_path, &nir_img_ptr, &rgb_img_ptr); if(ret != SDK_CODE_OK){ aw_loge("Fail to get test image data(%s and %s)\n", nir_img_path, rgb_img_path); goto read_face_image_pair_to_malloc_fail; } /* load model */ ret = read_file_to_malloc_heap(DETECT_MODEL_FILE_PATH, &face_model_config->detect_model_ptr, &face_model_config->detect_model_bytes); if(ret != SDK_CODE_OK){ aw_loge("Fail to read %s\n", DETECT_MODEL_FILE_PATH); goto read_file_malloc_heap_fail; } aw_logd("Face detect model bytes %d\n", face_model_config->detect_model_bytes); ret = read_file_to_malloc_heap(FACE_MODEL_FILE_PATH, &face_model_config->face_model_ptr, &face_model_config->face_model_bytes); if(ret != SDK_CODE_OK){ aw_loge("Fail to read %s\n", FACE_MODEL_FILE_PATH); goto read_file_malloc_heap_fail; } aw_logd("Face model bytes %d\n", face_model_config->face_model_bytes); ret = read_file_to_malloc_heap(FACE_LAZY_MODEL_FILE_PATH, &face_model_config->face_lazy_model_ptr, &face_model_config->face_lazy_model_bytes); if(ret != SDK_CODE_OK){ aw_loge("Fail to read %s\n", FACE_LAZY_MODEL_FILE_PATH); goto read_file_malloc_heap_fail; } aw_logd("Face lazy model bytes %d\n", face_model_config->face_lazy_model_bytes); /* init face detect model */ ret = pix_init_detect_model(&pix_auth, face_model_config->detect_model_ptr, face_model_config->detect_model_bytes); if(SDK_CODE_OK != ret){ aw_loge("Fail to pix_init_detect_model with %d\n", ret); goto pix_init_detect_model_fail; } aw_logd("success pix_init_detect_model\n"); /* run detect face */ pix_image_face_info_t *img1_face_info = (pix_image_face_info_t *)malloc(sizeof(pix_image_face_info_t)); ret = pix_commone_detect(nir_img_ptr, &img1_face_info->coord, &target_type); if(ret != SDK_CODE_OK){ aw_loge("Fail to pix_commone_detect with %d\n", ret); goto pix_commone_detect_fail; } aw_logd("target_type is %d\n", target_type); dump_pix_coord_info(&img1_face_info->coord); /* init face recognition model */ ret = pix_init_facekit_model(&pix_auth, face_model_config->face_model_ptr, face_model_config->face_model_bytes); if(SDK_CODE_OK != ret){ aw_loge("Fail to pix_init_facekit_model with %d\n", ret); goto pix_init_facekit_model_fail; } aw_logd("success pix_init_facekit_model : %s\n", FACE_MODEL_FILE_PATH); ret = pix_init_facekit_model_lazy_part(&pix_auth, face_model_config->face_lazy_model_ptr, face_model_config->face_lazy_model_bytes); if(SDK_CODE_OK != ret){ aw_loge("Fail to pix_init_facekit_model_lazy_part with %d\n", ret); goto pix_init_facekit_model_lazy_part_fail; } aw_logd("success pix_init_facekit_model_lazy_part\n"); /* register face info */ aw_logd("register face \n"); /* when register face info, is_register_flag should be 1 */ int is_register_flag = 1; /* TODO: register more faces */ face_model_config->register_img_face_info[0] = (pix_image_face_info_t *)malloc(sizeof(pix_image_face_info_t)); memset(face_model_config->register_img_face_info[0], 0, sizeof(pix_image_face_info_t)); ret = get_face_info(face_model_config->register_img_face_info[0], is_register_flag, nir_img_ptr, rgb_img_ptr); if (ret != SDK_CODE_OK){ aw_logd("Fail to get img1 face info\n"); free(face_model_config->register_img_face_info[0]); goto register_face_get_face_info_fail; } aw_logd("register face success\n"); register_face_get_face_info_fail: pix_init_facekit_model_lazy_part_fail: pix_init_facekit_model_fail: pix_commone_detect_fail: pix_init_detect_model_fail: read_file_malloc_heap_fail: if(face_model_config->detect_model_ptr != NULL) free(face_model_config->detect_model_ptr); if(face_model_config->face_model_ptr != NULL) free(face_model_config->face_model_ptr); if(face_model_config->face_lazy_model_ptr != NULL) free(face_model_config->face_lazy_model_ptr); read_face_image_pair_to_malloc_fail: if(nir_img_ptr != NULL) free(nir_img_ptr); if(rgb_img_ptr != NULL) free(rgb_img_ptr); return ret; } int rt_pixtalks_face_cal_fea_sim(struct rt_pixtalks_face_config_param* face_model_config, unsigned char *nir_ptr, unsigned char *rgb_ptr) { int ret = -1; /* when cal fea sim, is_register_flag should be 0 */ int is_register_flag = 0; pix_image_face_info_t *register_face_info = face_model_config->register_img_face_info[0]; pix_image_face_info_t *cur_img_face_info = (pix_image_face_info_t *)malloc(sizeof(pix_image_face_info_t)); ret = get_face_info(cur_img_face_info, is_register_flag, nir_ptr, rgb_ptr); aw_logd("dump cur face info:\n"); dump_pix_coord_info(cur_img_face_info); float pair_fr_score = 0; for(int i = 0; i < register_face_info->coord.box_number; i++){ for(int j = 0; j < cur_img_face_info->coord.box_number; j++){ ret = pix_cal_fea_sim(®ister_face_info->fr_feature[i], &cur_img_face_info->fr_feature[j], &pair_fr_score); if(SDK_CODE_OK != ret){ printf("Fail to pix_cal_fea_sim with %d\n", ret); break; } aw_logd("cur img vs register img fr score %f\n", pair_fr_score); } } return ret; } int rt_pixtalks_face_destroy(struct rt_pixtalks_face_config_param* face_model_config) { int ret = -1; for(int i=0; i< VALID_REGISTER_FACE_NUM ;i++) { if(face_model_config->register_img_face_info[i] != NULL) free(face_model_config->register_img_face_info[i]); if(face_model_config->nir_img_ptr[i] != NULL) free(face_model_config->nir_img_ptr[i]); if(face_model_config->rgb_img_ptr[i] != NULL) free(face_model_config->rgb_img_ptr[i]); } ret = pix_release_facekit_model(); if(ret != SDK_CODE_OK) { aw_loge("Fail to pix_release_facekit_model with %d, continue free resource\n", ret); } ret = vip_destroy(); if(ret != SDK_CODE_OK) { aw_loge("Fail to vip_destroy \n"); } return ret; }