001package net.gdface.sdk;
002
003import static gu.jimgutil.MatrixUtils.*;
004
005import java.io.IOException;
006import java.util.ArrayList;
007import java.util.Arrays;
008import java.util.Collections;
009import java.util.HashMap;
010import java.util.LinkedList;
011import java.util.List;
012import java.util.Map;
013import java.util.concurrent.atomic.AtomicBoolean;
014
015import net.gdface.image.ImageErrorException;
016import net.gdface.image.MatType;
017import net.gdface.image.BaseLazyImage;
018import net.gdface.image.UnsupportedFormatException;
019import net.gdface.sdk.fse.CodeBean;
020import net.gdface.sdk.fse.FeatureSe;
021import net.gdface.utils.Assert;
022import net.gdface.utils.ShareLock;
023
024/**
025 * {@link FaceApi}本地实现<br>
026 * SDK接口类本地实现必须从此类派生<br>
027 * @author guyadong
028 *
029 */
030public abstract class BaseFaceApiLocal extends BaseFaceApi implements CapacityFieldConstant {
031        private final FeatureSe fse;
032
033        protected final Map<String, String> capacity = new HashMap<>();
034        private final Map<String, String> roCapacity = Collections.unmodifiableMap(capacity);
035        /**
036         * 并发锁,限制建模的并发线程数,默认为CPU核数
037         */
038        protected static ShareLock concurrentLock = new ShareLock(Runtime.getRuntime().availableProcessors());
039        protected static IAuxTool auxTool = new IAuxTool() {            
040                @Override
041                public void saveOnFail(String phase, BaseLazyImage lazyImage) {
042                        // DO NOTHING
043                }
044        };
045        public static synchronized void setAuxTool(IAuxTool auxTool) {
046                if(null != auxTool){
047                        BaseFaceApiLocal.auxTool = auxTool;
048                }
049        }
050        protected BaseFaceApiLocal() {
051                capacity.put(C_MULTI_FACE_FEATURE, Boolean.FALSE.toString());
052                capacity.put(C_WEAR_MASK, Boolean.FALSE.toString());
053                fse = getFeatureSeInstance(this);               
054                capacity.put(C_FSE_ENABLE, fse == null ? Boolean.FALSE.toString(): Boolean.TRUE.toString());
055                capacity.put(C_CODEINFO_RELOCATE, Boolean.FALSE.toString());
056                capacity.put(C_LOCAL_DETECT, Boolean.TRUE.toString());
057        }
058        @Override
059        public CodeInfo[] detectAndGetCodeInfo(byte[] imgData, int faceNum) throws ImageErrorException, NotFaceDetectedException {
060                BaseLazyImage lazyImg = makeOpenedLazyImage(imgData);
061                try {
062                        return detectAndGetCodeInfo(lazyImg, faceNum);
063                } finally {
064                        try {
065                                lazyImg.close();
066                                lazyImg = null;
067                        } catch (IOException e) {
068                                throw new RuntimeException(e);
069                        }
070                }
071        }
072
073        /**
074         * 对图像({@code lazyImg})中指定的人脸位置{@code facePos}并提取特征码,返回生成人脸特征码数据数组对象(人脸可能不止一个)<br>
075         * 调用该方法时假设图像({@code lazyImage})能正常解码,所以当对图像解码出现异常时,将异常对象封装到{@link RuntimeException}抛出<br>
076         * 
077         * @param lazyImg
078         *            图像对象,为{@code null}则抛出 {@link IllegalArgumentException}
079         * @param faceNum
080         * @param facePos
081         * @return 返回{@link CodeInfo}对象列表<br>
082         *         对应于{@code facePos}中的每个{@link FRect}对象,返回一个{@link CodeInfo}对象(数组下标相同):<br>
083         *         如果{@link FRect}对象为{@code null},则返回一个所有成员都为{@code null}的{@link CodeInfo}对象<br>
084         *         如果在{@link FRect}指定的范围没有提取到特征码,则{@link CodeInfo#getCode()},{@link CodeInfo#getEi()}返回{@code null}
085         * @throws NotFaceDetectedException 
086         * @see #getCodeFromImageMatrix(byte[], int, int, CodeInfo, AtomicBoolean)
087         * @see FaceApi#getCodeInfo(byte[], int, CodeInfo[])
088         */
089        public List<CodeInfo> getCodeInfo(BaseLazyImage lazyImg, int faceNum, List<CodeInfo> facePos) throws NotFaceDetectedException {
090                concurrentLock.lock();
091                try {
092                        Assert.notNull(lazyImg, "lazyImg");
093                        byte[] imgData=lazyImg.getMatrixData(getNativeMatrixType());
094                        return nativeGetFaceFeatures(imgData, lazyImg.getWidth(), lazyImg.getHeight(),faceNum,facePos);
095                } catch (UnsupportedFormatException e) {
096                        throw new RuntimeException(e);
097                } finally {
098                        concurrentLock.unlock();
099                }       
100        }
101        /**
102         * 提取单个人脸特征 
103         * @param lazyImg
104         * @param facePos
105         * @return
106         */
107        public CodeInfo getCodeInfo(BaseLazyImage lazyImg, CodeInfo facePos){
108                concurrentLock.lock();
109                try {
110                        byte[] imgData=lazyImg.getMatrixData(getNativeMatrixType());
111                        byte[] feature = nativeGetFaceFeature(imgData, lazyImg.getWidth(), lazyImg.getHeight(), facePos);
112                        facePos.setCode(feature);
113                        return feature == null ? null : facePos;
114                } catch (UnsupportedFormatException e) {
115                        return null;
116                } finally {
117                        concurrentLock.unlock();
118                }
119        }
120        @Override
121        public CodeInfo getCodeInfo(byte[] imgData, CodeInfo facePos){
122                 try{
123                        BaseLazyImage lazyImg = makeOpenedLazyImage(imgData);
124                         return getCodeInfo(lazyImg, facePos);
125                 }catch (ImageErrorException e) {
126                         throw new RuntimeException(e);
127                }               
128        }
129
130        @Override
131        public Boolean wearMask(byte[] imgData, CodeInfo faceInfo) throws ImageErrorException{
132                BaseLazyImage lazyImg = makeOpenedLazyImage(imgData);
133                concurrentLock.lock();
134                try {
135                        return nativeWearMask(lazyImg.getMatrixData(getNativeMatrixType()), lazyImg.getWidth(), lazyImg.getHeight(), faceInfo);
136                } finally {
137                        try {
138                                lazyImg.close();
139                                lazyImg = null;
140                        } catch (IOException e) {
141                                throw new RuntimeException(e);
142                        }
143                        concurrentLock.unlock();
144                }
145        }
146        /**
147         * 先对图像数据{@code imgData}进行人脸检测,然后提取人脸特征码<br>
148         * 
149         * @param lazyImg
150         * @param faceNum
151         * @return
152         * @throws ImageErrorException
153         * @throws NotFaceDetectedException 
154         * @see #detectAndGetCodeInfo(byte[], int)
155         */
156        public CodeInfo[] detectAndGetCodeInfo(BaseLazyImage lazyImg, int faceNum) throws ImageErrorException,
157        NotFaceDetectedException {
158                concurrentLock.lock();
159                try{
160                        byte[] imgData = lazyImg.getMatrixData(getNativeMatrixType());
161                        CodeInfo[] codes = nativeDetectAndGetFeatures(imgData, lazyImg.getWidth(), lazyImg.getHeight());
162                        // 没有检测到要求的人脸则抛出异常
163                        if(0==codes.length){
164                                throw new NotFaceDetectedException();
165                        }
166                        if(faceNum>0&&faceNum!=codes.length){
167                                throw new NotFaceDetectedException();
168                        }
169                        return codes;
170                }finally{
171                        concurrentLock.unlock();
172                }               
173        }       
174
175        @Override
176        public CodeInfo[] detectFace(byte[] imgData) throws ImageErrorException {
177                BaseLazyImage lazyImg = makeOpenedLazyImage(imgData);
178                try {
179                        return detectFace(lazyImg).toArray(new CodeInfo[0]);
180                } finally {
181                        try {
182                                lazyImg.close();
183                                lazyImg = null;
184                        } catch (IOException e) {
185                                throw new RuntimeException(e);
186                        }
187                }
188        }
189
190        @Override
191        public CodeInfo[] getCodeInfo(byte[] imgData, int faceNum, CodeInfo[] facePos) throws NotFaceDetectedException {
192                BaseLazyImage lazyImg = null;
193                try {
194                        lazyImg = makeOpenedLazyImage(imgData);
195                        return getCodeInfo(lazyImg, faceNum, null == facePos ? null : Arrays.asList(facePos)).toArray(
196                                        new CodeInfo[0]);
197                } catch (ImageErrorException e) {
198                        throw new RuntimeException(e);
199                } finally {
200                        try {
201                                if (null != lazyImg) {
202                                        lazyImg.close();
203                                        lazyImg = null;
204                                }
205                        } catch (IOException e) {
206                                throw new RuntimeException(e);
207                        }
208                }
209        }
210
211        @Override
212        public final double compareCode(byte[] code1, byte[] code2) {
213                Assert.assertValidCode(code1, code2);
214                return nativeCompareCode(code1,code2);
215        }
216        @Override
217        public CodeInfo[] matDetectFace(MatType matType,byte[] matData, int width, int height){
218                byte[] nativeMat = toNativeMatrix(matType, matData, width, height);
219                concurrentLock.lock();
220                try {
221                        List<CodeInfo> faceInfo = new ArrayList<CodeInfo>();
222                        nativeDetectFace(nativeMat,width,height,faceInfo);
223                        return faceInfo.toArray(new CodeInfo[0]);
224                } finally {
225                        concurrentLock.unlock();
226                }
227        }
228        @Override
229        public CodeInfo[] matGetCodeInfo(MatType matType, byte[] matData, int width,int height, int facenum, CodeInfo[] facePos) throws NotFaceDetectedException{
230                byte[] nativeMat = toNativeMatrix(matType, matData, width, height);
231                Assert.notNull(facePos, "facePos");
232                concurrentLock.lock();
233                try {                   
234                        List<CodeInfo> codes = nativeGetFaceFeatures(nativeMat, width, height, facenum,Arrays.asList(facePos));
235                        return codes.toArray(new CodeInfo[codes.size()]);
236                } finally {
237                        concurrentLock.unlock();
238                }
239        }
240        @Override
241        public CodeInfo matGetCodeInfo(MatType matType, byte[] matData, int width,int height, CodeInfo facePos){
242                byte[] nativeMat = toNativeMatrix(matType, matData, width, height);
243                Assert.notNull(facePos, "facePos");
244                concurrentLock.lock();
245                try {                   
246                        byte[] feature = nativeGetFaceFeature(nativeMat, width, height, facePos);
247                        facePos.setCode(feature);
248                        return feature == null ? null : facePos;
249                } finally {
250                        concurrentLock.unlock();
251                }
252        }
253        @Override
254        public CodeInfo[] matDetectAndGetCodeInfo(MatType matType, byte[] matData, int width, int height, int faceNum) throws NotFaceDetectedException {
255                byte[] nativeMat = toNativeMatrix(matType, matData, width, height);
256                concurrentLock.lock();
257                try{
258                        CodeInfo[] codes = nativeDetectAndGetFeatures(nativeMat, width,height);
259                        // 没有检测到要求的人脸则抛出异常
260                        if(0==codes.length){
261                                throw new NotFaceDetectedException();
262                        }
263                        if(faceNum>0&&faceNum!=codes.length){
264                                throw new NotFaceDetectedException();
265                        }
266                        return codes;
267                }finally{
268                        concurrentLock.unlock();
269                }               
270        }
271        @Override
272        public Boolean matWearMask(MatType matType, byte[] matData, int width, int height, CodeInfo faceInfo){
273                byte[] nativeMat = toNativeMatrix(matType, matData, width, height);
274                concurrentLock.lock();
275                try {
276                        return nativeWearMask(nativeMat, width, height, faceInfo);
277                } finally {
278                        concurrentLock.unlock();
279                }
280        }
281        /**
282         * 返回当前{@link FaceApi}接口实例对应的搜索引擎接口实例,如果没有则返回{@code null}
283         * @return
284         */
285        public FeatureSe getFeatureSe(){
286                return null;
287        }
288        @Override
289        public boolean isLocal() {
290                return true;
291        }
292        @Override
293        public Map<String, String> sdkCapacity(){
294                return roCapacity;
295        }
296        /**
297         * 对图像({@code lazyImg})进行人脸检测,返回人脸位置数据对象{@link CodeInfo}列表<br>
298         * 
299         * @param lazyImg
300         *            图像对象,为{@code null}时抛出{@link IllegalArgumentException}
301         * @return 没有检测到人脸则返回空表
302         * @throws UnsupportedFormatException
303         */
304        public List<CodeInfo> detectFace(BaseLazyImage lazyImg)
305                        throws UnsupportedFormatException {
306                concurrentLock.lock();
307                try {
308                        Assert.notNull(lazyImg, "bufImg");
309                        List<CodeInfo> faceInfo = new ArrayList<CodeInfo>();
310                        if (lazyImg.getWidth() > 0 && lazyImg.getHeight() > 0) {
311                                byte[] imgRGB =lazyImg.getMatrixData(getNativeMatrixType());
312                                nativeDetectFace(imgRGB,lazyImg.getWidth(), lazyImg.getHeight(),faceInfo);
313                        }
314                        return faceInfo;
315                } finally {
316                        concurrentLock.unlock();
317                }
318        }
319        @Override
320        protected MatType getNativeMatrixType(){
321                return MatType.BGR;
322        }
323        @Override
324        protected byte[] toNativeMatrix(MatType matType, byte[] matData, int width, int height){
325                if(null == matData ){
326                        return null;
327                }
328                Assert.notNull(matType, "matType");
329                switch(matType){
330                case RGB:
331                        return RGB2BGR(matData, width, height, width*3);
332                case BGR:
333                        return matData;
334                case RGBA:
335                        return  RGBA2BGR(matData, width, height, width*4);
336                case NV21:
337                        return  NV212BGR(matData, width, height);
338                default:
339                        throw new IllegalArgumentException("UNSUPPORTED TARGET MATRIX TYPE " + matType + " for NV21 CONVERTING");
340                }
341        }
342        /**
343         * 参见 {@link #compareCode(byte[], byte[])}
344         * @param code1
345         * @param code2
346         * @return
347         */
348        protected abstract double nativeCompareCode(byte[] code1, byte[] code2) ;
349        
350        /**
351         * 检测人脸
352         * @param imgMatrix 图像矩阵数据,数据格式要求与算法相关
353         * @param width 图像宽度
354         * @param height 图像高度
355         * @param faceInfo 保存检测到的人脸位置信息
356         */
357        public abstract void nativeDetectFace(byte[] imgMatrix, int width, int height, List<CodeInfo> faceInfo);
358        /**
359         * 对faceinfo指定的人脸位置提取特征
360         * @param imgMatrix 图像矩阵数据,数据格式要求与算法相关
361         * @param width 图像宽度
362         * @param height 图像高度
363         * @param faceInfo
364         * @return 人脸特征数据,提取特征失败则返回null
365         */
366        public abstract byte[] nativeGetFaceFeature(byte[] imgMatrix, int width, int height, CodeInfo faceInfo);
367        /**
368         * 对图像矩阵中指定的人脸位置{@code facePos}并提取特征码,返回生成人脸特征码数据数组对象(人脸可能不止一个)<br>
369         * 
370         * @param imgMatrix 图像矩阵数据,数据格式要求与算法相关
371         * @param width 图像宽度
372         * @param height 图像高度
373         * @param faceNum
374         * @param facePos
375         * @return 返回{@link CodeInfo}对象列表<br>
376         *         对应于{@code facePos}中的每个{@link FRect}对象,返回一个{@link CodeInfo}对象(数组下标相同):<br>
377         *         如果{@link FRect}对象为{@code null},则返回一个所有成员都为{@code null}的{@link CodeInfo}对象<br>
378         *         如果在{@link FRect}指定的范围没有提取到特征码,则{@link CodeInfo#getCode()},{@link CodeInfo#getEi()}返回{@code null}
379         * @throws NotFaceDetectedException 
380         * @see {@link #nativeGetFaceFeatures(byte[], int, int, List)}
381         */
382        public List<CodeInfo> nativeGetFaceFeatures(byte[] imgMatrix, int width, int height,int faceNum, List<CodeInfo> facePos) throws NotFaceDetectedException {
383                // 检测到的人脸数目
384                int faceCount = 0;
385                // 提取到特征码的人脸数目
386                int featureCount=0;
387                Assert.notNull(imgMatrix, "imgMatrix");
388                // facePos为null或空时抛出异常
389                Assert.notEmpty(facePos, "facePos");
390                // 人脸位置对象数目小于要求提取的特征码的数目,则抛出异常
391                if (facePos.size() < faceNum){
392                        throw new NotFaceDetectedException(facePos.size(), 0);
393                }
394                byte[][] features = nativeGetFaceFeatures(imgMatrix, width, height,facePos);
395                for(int i = 0 ;i<features.length;++i){
396                        if(null!=facePos.get(i)){
397                                ++faceCount;
398                                if(null!=features[i]){
399                                        ++featureCount;
400                                }
401                        }
402                }
403                // 提取到的人脸特征数目不等于要求提取的特征码的数目,则抛出异常
404                if ((faceNum > 0)) {
405                        if (featureCount != faceNum){
406                                throw new NotFaceDetectedException(faceCount, featureCount);
407                        }
408                } else if (0 == featureCount){
409                        throw new NotFaceDetectedException(faceCount, 0);
410                }
411                for(int i=0;i<facePos.size();++i){
412                        CodeInfo ci = facePos.get(i);
413                        if(null!=ci){
414                                ci.setCode(features[i]);
415                        }
416                }
417                return facePos;
418
419        }
420        /**
421         * 根据检测到的人脸位置{@code facePos},提取一组特征
422         * @param imgMatrix 图像矩阵数据,数据格式要求与算法相关
423         * @param width 图像宽度
424         * @param height 图像高度
425         * @param facePos
426         * @return 返回facePos列表中每个人脸位置对应的人脸特征数据,提取特征失败的对应为null
427         */
428        public byte[][] nativeGetFaceFeatures(byte[] imgMatrix, int width, int height, List<CodeInfo> facePos) {
429                byte[][] res = new byte[facePos.size()][];
430                for(int i=0;i<facePos.size();++i){
431                        CodeInfo face = facePos.get(i);
432                        if(null != face){
433                                res[i]=nativeGetFaceFeature(imgMatrix, width, height, face);
434                                face.setCode(res[i]);
435                        }else{
436                                res[i] = null;
437                        }
438                }               
439                return res;                     
440        }
441        /**
442         * 检测并提取特征
443         * @param imgMatrix 图像矩阵数据,数据格式要求与算法相关
444         * @param width 图像宽度
445         * @param height 图像高度
446         * @return 包含人脸特征数据的{@code CodeInfo}数组
447         */
448        public CodeInfo[] nativeDetectAndGetFeatures(byte[] imgMatrix, int width, int height) {
449                LinkedList<CodeInfo> faceInfo = new LinkedList<CodeInfo>();
450                nativeDetectFace(imgMatrix, width, height, faceInfo);
451                for( CodeInfo face:faceInfo){
452                        face.setCode(nativeGetFaceFeature(imgMatrix, width, height, face));
453                }
454                return faceInfo.toArray(new CodeInfo[faceInfo.size()]);
455        }
456        /**
457         * 对从视频流中直接获取的NV21图像进行人脸检测
458         * @param nv21 NV21格式图像
459         * @param width 图像宽度
460         * @param height 图像高度
461         * @param faceInfo 人脸信息对象列表
462         */
463        public void nativeNv21DetectFace(byte[] nv21,int width, int height,List<CodeInfo> faceInfo){
464                byte[] matrixData = toNativeMatrix(MatType.NV21, nv21, width, height);
465                if (width > 0 && height > 0) {
466                        nativeDetectFace(matrixData,width, height,faceInfo);
467                }
468        }
469
470        /**
471         * 根据facePos提供的人脸信息位置, 在NV21格式图像中提取特征码
472         * @param nv21 NV21格式图像
473         * @param width 图像宽度
474         * @param height 图像高度
475         * @param facePos 人脸位置信息对象,不可为{@code null}
476         * @return 包含人脸特征数据的{@code CodeInfo}对象
477         */
478        public byte[] nativeNv21GetFaceFeature(byte[] nv21, int width, int height,CodeInfo facePos){
479                Assert.notNull(facePos, "facePos");
480                byte[] matrixData = toNativeMatrix(MatType.NV21, nv21, width, height);
481                return nativeGetFaceFeature(matrixData,width, height, facePos);
482        }
483        /**
484         * 根据facePos提供的人脸信息位置, 在NV21格式图像中提取特征码
485         * @param nv21 NV21格式图像
486         * @param width 图像宽度
487         * @param height 图像高度
488         * @param facePos 人脸位置信息对象,不可为{@code null}
489         * @return 包含人脸特征数据的{@code CodeInfo}对象
490         */
491        public byte[][] nativeNv21GetFaceFeatures(byte[] nv21, int width, int height,List<CodeInfo> facePos){
492                Assert.notNull(facePos, "facePos");
493                byte[] matrixData = toNativeMatrix(MatType.NV21, nv21, width, height);
494                return nativeGetFaceFeatures(matrixData,width, height, facePos);
495        }
496        /**
497         * 检测并提取特征
498         * @param nv21 NV21格式图像矩阵数据
499         * @param width 图像宽度
500         * @param height 图像高度
501         * @return 包含人脸特征数据的{@code CodeInfo}数组
502         */
503        public CodeInfo[] nativeNv21DetectAndGetFeatures(byte[] nv21, int width, int height) {
504                byte[] matrixData = toNativeMatrix(MatType.NV21, nv21, width, height);
505                CodeInfo[] faceInfo = nativeDetectAndGetFeatures(matrixData, width, height);
506                for( CodeInfo face:faceInfo){
507                        face.setCode(nativeGetFaceFeature(nv21, width, height, face));
508                }
509                return faceInfo;
510        }
511        /**
512     * 戴口罩检测检测方法
513     *
514     * @param imgMatrix   图像矩阵数据,数据格式要求与算法相关
515     * @param width  图像宽度
516     * @param height 图像高度
517     * @param faceInfo  可见光图像人脸信息
518     * @return 戴口罩返回{@code true},否则返回{@code false},{@code null} 不知道
519     */
520        public Boolean nativeWearMask(byte[] imgMatrix, int width, int height, CodeInfo faceInfo){
521                return null;
522        }
523        /**
524     * 戴口罩检测检测方法
525     *
526     * @param nv21   NV21格式图像矩阵数据
527     * @param width  图像宽度
528     * @param height 图像高度
529     * @param faceInfo  可见光图像人脸信息
530     * @return 戴口罩返回{@code true},否则返回{@code false},{@code null} 不知道
531     */
532        public Boolean nativeNv21WearMask(byte[] nv21, int width, int height, CodeInfo faceInfo){
533                return null;
534        }
535        
536        @Override
537        public FseResult[] searchFeatures(byte[] feature, double similarty, int rows) {
538                if(null != fse){
539                        return CodeBean.toFseResult(fse.searchCode(feature, similarty, rows));
540                }
541                throw new UnsupportedOperationException();
542        }
543        protected static byte[] fromNv21(MatType matType, byte[] nv21, int width, int height) {
544                Assert.notNull(matType, "matType");
545                Assert.notNull(nv21, "nv21");
546                switch(matType){
547                case RGB:
548                        return NV212RGB(nv21, width, height);
549                case BGR:
550                        return NV212BGR(nv21, width, height);
551                case RGBA:
552                        return  NV212RGBA(nv21, width, height);
553                default:
554                        throw new IllegalArgumentException("UNSUPPORTED TARGET MATRIX TYPE " + matType + " for NV21 CONVERTING");
555                }
556        }
557        protected static byte[] toNv21(MatType matType,byte[] matData, int width, int height){
558                Assert.notNull(matType, "matType");
559                Assert.notNull(matData, "matData");
560                switch (matType) {
561                case NV21:
562                        return matData;
563                case RGB:
564                        return RGB2NV21(matData, width, height);
565                case BGR:
566                        return BGR2NV21(matData, width, height);
567                case RGBA:
568                        return RGBA2NV21(matData, width, height);
569                default:
570                        throw new UnsupportedOperationException("UNSUPPORTED INPUT MATRIX TYPE: " + matType);
571                }
572        }
573}