结合骨干模型和手工特征的混合深度学习模型

Question

我有 RGB 图像，我想构建一个回归模型来预测“Lodging_score”，结合 dendestnet121 作为主干和 csv 文件中的手工特征。运行下面的脚本，出现以下错误 ValueError: Layer "model" Expects 2 input(s), but it receive 1 input tensors.收到的意见：[]。如果您能帮助我，我将不胜感激，我已经挣扎了好几天了。

#Step 1: Import the required libraries  
import tensorflow as tf
from tensorflow.keras.applications import DenseNet121
from tensorflow.keras.layers import Dense, Dropout, Input, Concatenate, GlobalAveragePooling2D
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
import pandas as pd
import numpy as np

modelID = 'd121_HCF'

#Step 2: Load and preprocess the image data 
image_dir = r'/path_to_images_folder'
annotations_file = '/path_to/annotation.csv'
features_file = 'handcrafted_features.csv'

# Load image filenames and labels from annotations file
annotations_df = pd.read_csv(annotations_file)

image_filenames = annotations_df['Image_filename'].tolist()
labels = annotations_df['Lodging_score'].tolist()

# Load handcrafted features
features_df = pd.read_csv(features_file)
features_df.set_index('Image_filename', inplace=True)

# Get common image filenames
common_filenames = list(set(image_filenames).intersection(features_df.index))
#print(len(common_filenames))

# Filter the annotation and feature dataframes based on common filenames
annotations_df = annotations_df[annotations_df['Image_filename'].isin(common_filenames)]
features_df = features_df.loc[common_filenames]
features_df = features_df.drop(columns=['plot_id','project_id','Lodging_score'])# dropping columns that are not features

# Split the data into train, val, and test sets
train_filenames, test_filenames, train_labels, test_labels = train_test_split(
    annotations_df['Image_filename'].tolist(),
    annotations_df['Lodging_score'].tolist(),
    test_size=0.2,
    random_state=42)

val_filenames, test_filenames, val_labels, test_labels = train_test_split(
    test_filenames,
    test_labels,
    test_size=0.5,
    random_state=42)

# Preprocess handcrafted features
train_features = features_df.loc[train_filenames].values
val_features = features_df.loc[val_filenames].values
test_features = features_df.loc[test_filenames].values

# Normalize handcrafted features
train_features = (train_features - train_features.mean(axis=0)) / train_features.std(axis=0)
val_features = (val_features - train_features.mean(axis=0)) / train_features.std(axis=0)
test_features = (test_features - train_features.mean(axis=0)) / train_features.std(axis=0)

# Convert the label arrays to numpy arrays
train_labels = np.array(train_labels)
val_labels = np.array(val_labels)
test_labels = np.array(test_labels)

# Preprocess handcrafted features
train_features = train_features[:len(train_filenames)]
val_features = val_features[:len(val_filenames)]
test_features = test_features[:len(test_filenames)]

# Define image data generator with augmentations
image_size = (75, 200)
batch_size = 32

image_data_generator = ImageDataGenerator(
    rescale=1./255,
    rotation_range=20,
    width_shift_range=0.1,
    height_shift_range=0.1,
    horizontal_flip=True)

train_data = pd.DataFrame({'filename': train_filenames, 'Lodging_score': train_labels})
train_generator = image_data_generator.flow_from_dataframe(
    train_data,
    directory=image_dir,
    x_col='filename',
    y_col='Lodging_score',
    target_size=image_size,
    batch_size=batch_size,
    class_mode='raw',
    shuffle=False)

val_generator = image_data_generator.flow_from_dataframe(
    pd.DataFrame({'filename': val_filenames, 'Lodging_score': val_labels}),
    directory=image_dir,
    x_col='filename',
    y_col='Lodging_score',
    target_size=image_size,
    batch_size=batch_size,
    class_mode='raw',
    shuffle=False)

# Create test generator
test_generator = image_data_generator.flow_from_dataframe(
    pd.DataFrame({'filename': test_filenames, 'Lodging_score': test_labels}),
    directory=image_dir,
    x_col='filename',
    y_col='Lodging_score',
    target_size=image_size,
    batch_size=batch_size,  # Keep the batch size the same as the other generators
    class_mode='raw',
    shuffle=False)

#Step 3: Build the hybrid regression model
# Load DenseNet121 pre-trained on ImageNet without the top layer
base_model = DenseNet121(include_top=False, weights='imagenet', input_shape=image_size + (3,))

# Freeze the base model's layers
base_model.trainable = False

# Input layers for image data and handcrafted features
image_input = Input(shape=image_size + (3,))
features_input = Input(shape=(train_features.shape[1],))

# Preprocess image input for DenseNet121
image_preprocessed = tf.keras.applications.densenet.preprocess_input(image_input)

# Extract features from the base model
base_features = base_model(image_preprocessed, training=False)
base_features = GlobalAveragePooling2D()(base_features)

# Combine base model features with handcrafted features
combined_features = Concatenate()([base_features, features_input])

# Add dense layers for regression
x = Dropout(0.5)(combined_features)
x = Dense(128, activation='relu')(x)
x = Dropout(0.5)(x)
output = Dense(1, activation='linear')(x)

# Create the model
model = Model(inputs=[image_input, features_input], outputs=output)

# Compile the model
model.compile(optimizer=Adam(learning_rate=0.001), loss='mean_squared_error')

#Step 4: Train the model with early stopping   
# Define early stopping callback
early_stopping = tf.keras.callbacks.EarlyStopping(
    monitor='val_loss', patience=5, restore_best_weights=True)

# Convert numpy arrays to tensors
train_features_tensor = tf.convert_to_tensor(train_features, dtype=tf.float32)
val_features_tensor = tf.convert_to_tensor(val_features, dtype=tf.float32)
test_features_tensor = tf.convert_to_tensor(test_features, dtype=tf.float32)

# Train the model
history = model.fit(
    train_generator,
    steps_per_epoch=len(train_generator),
    epochs=50,
    validation_data=([val_generator.next()[0], val_features], val_labels),
    validation_steps=len(val_generator),
    callbacks=[early_stopping])

# Evaluate the model on the test set
loss = model.evaluate([test_generator.next()[0], test_features], test_labels, verbose=0)
predictions = model.predict([test_generator.next()[0], test_features])

Answer 1

查看

tf.Dataset

。 在这里您可以看到如何从 DataFrame 中读取它们。然后，可以使用

Dataset.map()

进行预处理，将预处理函数映射到每个元素上，或者使用像本指南中的 那样的预处理层。您也可以使用这些层进行增强。

因为你有两个输入层，所以你的数据也必须有这个输入。示例代码：

import numpy as np
import pandas as pd
import tensorflow as tf


img_paths = ['test'] * 100  # 100 image paths here
rand_features = np.random.rand(100, 3)  # random features
rand_labels = np.random.randint(0, 10, size=(100, 1))  # random labels as int

#right now, the dataset has a 3-tuple as samples
ds = tf.data.Dataset.from_tensor_slices((img_paths, rand_features, rand_labels))
ds = ds.map(lambda x, y, z: ((tf.image.decode_image(tf.io.read_file(x)), y), z))
ds = ds.batch(32)  # batch the data

在

ds.map(...)

 调用中，对于每个元组元素，它从 

image_path

 字符串中读取图像，并将 

(image, feature, label)

 3 元组转换为 

((image, feature), label)

 嵌套 2 元组。现在，model.fit 可以接受每个 

(image, feature)

 作为 2 个输入 (x)，以及 

label

 作为 (y)。将其放在图像和从数据帧读取的特征之后。您还可以查看其他方便的 

Dataset

 方法，例如 

.prefetch() 或 .shuffle()

。