我们正在制作一个使用 Firebase ML Kit 调用自定义模型的应用程序。我们已经注册了yolo自定义模型，并且连接正常。然而，Firebase提供的模型结果不断出来，而不是我们指定的自定义模型结果。不断修改代码也会发生同样的情况，此时我可以检查代码的哪一部分？
`plugins {
id 'com.android.application'
id 'com.google.gms.google-services'
id 'org.jetbrains.kotlin.android'
}
android {
namespace 'com.example.wishwash'
compileSdk 33
viewBinding {
enabled = true
}
defaultConfig {
applicationId "com.example.wishwash"
minSdk 23
targetSdk 33
versionCode 1
versionName "1.0"
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
}
buildTypes {
release {
minifyEnabled false
proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
}
}
compileOptions {
sourceCompatibility JavaVersion.VERSION_1_8
targetCompatibility JavaVersion.VERSION_1_8
}
sourceSets { main { res.srcDirs = ['src/main/res', 'src/main/res/raw'] } }
/*kotlinOptions {
jvmTarget = '1.8'
}*/
}
dependencies {
implementation 'androidx.swiperefreshlayout:swiperefreshlayout:1.1.0'
implementation 'androidx.appcompat:appcompat:1.6.1'
implementation 'com.google.android.material:material:1.9.0'
implementation 'androidx.annotation:annotation:1.6.0'
implementation 'androidx.constraintlayout:constraintlayout:2.1.4'
implementation platform('com.google.firebase:firebase-bom:32.0.0')
implementation 'com.google.firebase:firebase-analytics'
//implementation 'com.kakao.sdk:v2-map:1.4.2@aar'
//implementation 'net.daum.mf:map-sdk:4.5.0'
// implementation 'com.google.mlkit:image-labeling:17.0.7'
implementation 'com.google.code.gson:gson:2.8.6'
implementation 'com.google.mlkit:image-labeling-custom:17.0.1'
implementation 'com.google.mlkit:linkfirebase:17.0.0'
implementation 'com.google.firebase:firebase-ml-vision-object-detection-model:20.1.3'
//implementation 'com.google.firebase:firebase-ml-custom:22.0.0'
implementation 'com.google.firebase:firebase-ml-common:22.1.2'
implementation 'com.google.firebase:firebase-ml-modeldownloader'
implementation 'com.google.firebase:firebase-ml-vision-image-label-model:20.0.2'
// implementation 'com.google.firebase:firebase-ml-model-interpreter:22.0.4'
implementation 'org.tensorflow:tensorflow-lite:2.6.0'
implementation 'com.google.firebase:firebase-core:21.1.1'
implementation 'com.google.android.gms:play-services-vision:20.1.3'
implementation 'com.google.android.gms:play-services-vision-common:19.1.3'
implementation 'com.google.android.gms:play-services-vision-image-labeling-internal:16.1.0'
// def camerax_version = "1.3.0-alpha05"
// The following line is optional, as the core library is included indirectly by camera-camera2
implementation 'androidx.camera:camera-core:1.2.3'
implementation "androidx.camera:camera-camera2:1.2.3"
// If you want to additionally use the CameraX Lifecycle library
implementation "androidx.camera:camera-lifecycle:1.2.3"
// If you want to additionally use the CameraX View class
implementation "androidx.camera:camera-view:1.2.3"
implementation 'androidx.constraintlayout:constraintlayout:2.1.4'
//implementation 'com.google.firebase:firebase-model-downloader:29.0.0'
//noinspection OutdatedLibrary
implementation 'com.google.firebase:firebase-ml-vision:24.1.0'
implementation 'com.google.mlkit:image-labeling-default-common:17.0.0'
testImplementation 'junit:junit:4.13.2'
implementation fileTree(dir: 'src\\main\\jniLibs', include: ['*.aar', '*.jar'], exclude: [])
androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'
androidTestImplementation 'androidx.test:runner:1.5.2'
androidTestImplementation 'androidx.test.ext:junit:1.1.5'
androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'
implementation fileTree(include: ['*.jar'], dir: 'libs')
implementation files('libs/libDaumMapAndroid.jar')
}`
package com.example.wishwash;
import android.Manifest;
import android.annotation.SuppressLint;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.content.res.AssetFileDescriptor;
import android.graphics.Bitmap;
import android.graphics.Color;
import android.os.Bundle;
import android.util.Log;
import android.widget.Button;
import androidx.activity.result.ActivityResultLauncher;
import androidx.activity.result.contract.ActivityResultContracts;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.appcompat.app.AppCompatActivity;
import androidx.camera.core.Camera;
import androidx.camera.core.CameraSelector;
import androidx.camera.core.ExperimentalGetImage;
import androidx.camera.core.ImageAnalysis;
import androidx.camera.core.ImageProxy;
import androidx.camera.core.Preview;
import androidx.camera.lifecycle.ProcessCameraProvider;
import androidx.camera.view.PreviewView;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.firebase.ml.modeldownloader.CustomModelDownloadConditions;
import com.google.firebase.ml.modeldownloader.DownloadType;
import com.google.firebase.ml.modeldownloader.FirebaseModelDownloader;
import com.google.mlkit.vision.common.InputImage;
import com.google.mlkit.vision.label.ImageLabel;
import com.google.mlkit.vision.label.ImageLabeler;
import com.google.mlkit.vision.label.ImageLabeling;
import com.google.mlkit.vision.label.defaults.ImageLabelerOptions;
import org.tensorflow.lite.Interpreter;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@ExperimentalGetImage
public class CameraActivity extends AppCompatActivity {
private static final int REQUEST_CODE_PERMISSIONS = 101;
private static final String[] REQUIRED_PERMISSIONS = {Manifest.permission.CAMERA};
private static final String TAG = "CameraActivity";
private PreviewView previewView;
private ActivityResultLauncher<Intent> cameraLauncher;
//private CustomModel model;
private ExecutorService cameraExecutor;
private ImageLabeler labeler;
private Interpreter interpreter;
@SuppressLint("MissingInflatedId")
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_camera);
previewView = findViewById(R.id.previewView);
Button btn_photo = findViewById(R.id.btn_photo);
try {
AssetFileDescriptor assetFileDescriptor = getAssets().openFd("wishwash.tflite");
FileInputStream fileInputStream = new FileInputStream(assetFileDescriptor.getFileDescriptor());
FileChannel fileChannel = fileInputStream.getChannel();
long startOffset = assetFileDescriptor.getStartOffset();
long declaredLength = assetFileDescriptor.getDeclaredLength();
MappedByteBuffer modelBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
// 모델을 Interpreter로 로드
interpreter = new Interpreter(modelBuffer);
} catch (IOException e) {
// 모델 로드 중 오류 발생
e.printStackTrace();
}
if (checkCameraPermission()) {
startCamera();
} else {
requestCameraPermission();
}
cameraLauncher = registerForActivityResult(new ActivityResultContracts.StartActivityForResult(),
result -> {
if (result.getResultCode() == RESULT_OK) {
Intent data = result.getData();
if (data != null) {
Bitmap photo = (Bitmap) data.getExtras().get("data");
processPhoto(photo);
}
}
});
btn_photo.setOnClickListener(v -> {
Intent cameraIntent = new Intent(android.provider.MediaStore.ACTION_IMAGE_CAPTURE);
cameraLauncher.launch(cameraIntent);
});
}
private boolean checkCameraPermission() {
return ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED;
}
private void startCamera() {
ListenableFuture<ProcessCameraProvider> cameraProviderFuture = ProcessCameraProvider.getInstance(this);
cameraProviderFuture.addListener(() -> {
try {
ProcessCameraProvider cameraProvider = cameraProviderFuture.get();
// Custom Model 다운로드 및 설정
CustomModelDownloadConditions conditions = new CustomModelDownloadConditions.Builder()
.requireWifi()
.build();
FirebaseModelDownloader.getInstance()
.getModel("wishwash", DownloadType.LOCAL_MODEL_UPDATE_IN_BACKGROUND, conditions)
.addOnSuccessListener(model -> {
// Set the ImageLabelerOptions.
ImageLabelerOptions options = new ImageLabelerOptions.Builder()
.setExecutor(cameraExecutor)
.setConfidenceThreshold(0.5f)
.build();
labeler = ImageLabeling.getClient(options);
bindCameraPreview(cameraProvider); // 이미지 분석기 설정과 함께 bindCameraPreview 메서드를 호출합니다.
Log.d(TAG, "Model download sucess.");
})
.addOnFailureListener(e -> {
// 모델 다운로드가 실패하였습니다. 에러 처리를 수행합니다.
Log.e(TAG, "Model download failed.", e);
});
} catch (ExecutionException | InterruptedException e) {
Log.e(TAG, "Error binding camera.", e);
}
}, ContextCompat.getMainExecutor(this));
cameraExecutor = Executors.newSingleThreadExecutor();
}
@SuppressLint("SetTextI18n")
private void bindCameraPreview(ProcessCameraProvider cameraProvider) {
Preview preview = new Preview.Builder().build();
CameraSelector cameraSelector = new CameraSelector.Builder()
.requireLensFacing(CameraSelector.LENS_FACING_BACK)
.build();
ImageAnalysis imageAnalysis = new ImageAnalysis.Builder()
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.build();
// 이미지 분석기 설정
imageAnalysis.setAnalyzer(cameraExecutor, imageProxy -> {
Bitmap bitmap = imageProxyToBitmap(imageProxy);
if (bitmap != null) {
Bitmap scaledBitmap = Bitmap.createScaledBitmap(bitmap, 224, 224, true);
ByteBuffer inputBuffer = ByteBuffer.allocateDirect(224 * 224 * 3 * 4).order(ByteOrder.nativeOrder());
for (int y = 0; y < 224; y++) {
for (int x = 0; x < 224; x++) {
int px = scaledBitmap.getPixel(x, y);
// Get channel values from the pixel value.
int r = Color.red(px);
int g = Color.green(px);
int b = Color.blue(px);
// Normalize channel values to [-1.0, 1.0].
float rf = (r - 127) / 255.0f;
float gf = (g - 127) / 255.0f;
float bf = (b - 127) / 255.0f;
inputBuffer.putFloat(rf);
inputBuffer.putFloat(gf);
inputBuffer.putFloat(bf);
}
}
labeler.process(InputImage.fromBitmap(bitmap, 0))
.addOnSuccessListener(labels -> {
if (labels.isEmpty()) {
// 결과가 없을 경우 처리할 작업을 여기에 추가하세요.
// 예를 들어, 사용자에게 결과가 없음을 알리는 메시지를 표시하거나
// 기본값을 사용하여 특정 작업을 수행할 수 있습니다.
Log.d(TAG, "레이블링 결과 없음");
} else {
for (ImageLabel label : labels) {
label.getText();
label.getConfidence();
String labelText = label.getText();
String confidenceText = String.valueOf(label.getConfidence());
getString(R.string.label_confidence, labelText, confidenceText);
}
Log.d(TAG, "레이블링 결과 있음");
}
})
.addOnFailureListener(e -> Log.e(TAG, "레이블링 중 오류 발생.", e))
.addOnCompleteListener(task -> imageProxy.close()); // 이미지를 처리한 후에 이미지를 해제합니다.
} else {
imageProxy.close(); // 이미지를 처리하지 못한 경우에도 이미지를 해제합니다.
}
});
// CameraProvider에 미리보기 및 이미지 분석기를 바인딩합니다.
Camera camera = cameraProvider.bindToLifecycle(this, cameraSelector, preview, imageAnalysis);
preview.setSurfaceProvider(previewView.getSurfaceProvider());
}
@Nullable
private Bitmap imageProxyToBitmap(ImageProxy imageProxy) {
Bitmap bitmap = null;
if (imageProxy == null) {
return null;
}
try {
ImageProxy.PlaneProxy[] planes = imageProxy.getPlanes();
if (planes.length > 0) {
ImageProxy.PlaneProxy plane = planes[0];
ByteBuffer buffer = plane.getBuffer();
int pixelStride = plane.getPixelStride();
int rowStride = plane.getRowStride();
int rowPadding = rowStride - pixelStride * imageProxy.getWidth();
// 비트맵 크기 계산
int width = imageProxy.getWidth();
int height = imageProxy.getHeight();
// 버퍼 유효성 검사 및 조정
int requiredSize = width * height * 4; // 필요한 크기 계산
// 버퍼의 용량이 충분한지 확인
if (buffer.capacity() >= requiredSize) {
// 충분한 크기를 가지고 있으므로 해당 버퍼를 계속 사용할 수 있습니다.
buffer.rewind(); // 버퍼 포인터를 처음으로 되돌립니다.
} else {
// 충분한 크기를 가지고 있지 않으므로 버퍼를 재할당합니다.
buffer = ByteBuffer.allocate(requiredSize);
}
// 비트맵 생성
bitmap = Bitmap.createBitmap(width + rowPadding / pixelStride, height, Bitmap.Config.ARGB_8888);
bitmap.copyPixelsFromBuffer(buffer);
}
} catch (Exception e) {
Log.e(TAG, "ImageProxy를 Bitmap으로 변환하는 중 오류가 발생했습니다.", e);
} finally {
// 이미지 해제
imageProxy.close();
}
return bitmap;
}
private void requestCameraPermission() {
ActivityCompat.requestPermissions(this, REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS);
}
@Override
public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
if (requestCode == REQUEST_CODE_PERMISSIONS) {
if (grantResults.length > 0 && grantResults[0] == PackageManager.PERMISSION_GRANTED) {
startCamera();
} else {
Log.e(TAG, "Camera permission denied.");
}
}
}
@Override
protected void onDestroy() {
super.onDestroy();
cameraExecutor.shutdown();
}
private void processPhoto(Bitmap photo) {
int width = photo.getWidth();
int height = photo.getHeight();
Log.d(TAG, "Photo width: " + width + ", height: " + height);
InputImage image = InputImage.fromBitmap(photo, 0);
Object[] input = new Object[]{image};
int bufferSize = 1000 * java.lang.Float.SIZE / java.lang.Byte.SIZE;
ByteBuffer modelOutput = ByteBuffer.allocateDirect(bufferSize).order(ByteOrder.nativeOrder());
interpreter.run(input, modelOutput);
modelOutput.rewind();
FloatBuffer probabilities = modelOutput.asFloatBuffer();
try {
BufferedReader reader = new BufferedReader(
new InputStreamReader(getAssets().open("yolov8n.txt")));
for (int i = 0; i < probabilities.capacity(); i++) {
String label = reader.readLine();
float probability = probabilities.get(i);
Log.i(TAG, String.format("%s: %1.4f", label, probability));
}
} catch (IOException e) {
// File not found?
}
}
}