2019FastDepth论文复现（tensorflow）

论文笔记

简介

笔记（这个部分以后需要边看边写）

摘要

• 嵌入式系统上的快速深度估计的问题

• 提出了一种高效和轻量级的编码器-解码器网络，解码器经过特别设计了，低延迟

• 跟之前baseline比精度近似，快一个数量级（sota）

• 实验证明帧率很高

之前的工作

Introduction

• 编码器-解码器结构，编码器用MobileNet，解码器用的深度可分离卷积的最近邻内插

• 手工设计网络参数冗余，训练时剪枝了

• 因为当时嵌入式平台没有对MobileNet的可分离卷积作优化，还对这个用TVM编译器做了优化

网络主体

复现

先直接上代码

from tensorflow.keras.layers import Conv2D, UpSampling2D, SeparableConv2D, BatchNormalization, Activation, add
from tensorflow.keras.models import Model
from tensorflow.keras import optimizers
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.applications.mobilenet import MobileNet
# 加载.h5图片数据
import numpy as np
from tensorflow.keras.utils import Sequence
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Layer
import cv2
import h5py as h5
import datetime
import os

""" DataGenerator初始化"""
input_size=(224,224)
shuffle=True
train_txt_path='/data0/lijiaqi/traindata/nyudepthv2/train.txt'
val_txt_path='/data0/lijiaqi/traindata/nyudepthv2/val.txt'

# 训练时每个epoch中的step数量，在训练集DataGenerator的init函数中自动赋值，无需更改
train_steps_per_epoch=10
val_steps_per_epoch=10

class DataGenerator(Sequence):
    """
    基于Sequence的自定义Keras数据生成器
    """

    def __init__(self, txt_path , to_fit, batch_size=5, shuffle=shuffle):
        """

        :rtype: object
        """
        # 初始化方法
        # :param data_txt_path: 存h5数据路径的txt的所在位置
        # :param data_path_list:     h5数据路径列表
        # :param to_fit: 设定是否返回结果y
        # :param batch_size: batch size
        # :param shuffle: 每一个epoch后是否打乱数据

        self.data_txt_path = txt_path
        self.data_path_list=open(txt_path, 'r').readlines()
        # data_num_all 训练集/验证集中的数据个数
        self.data_num_all=len(self.data_path_list)
        self.batch_size = batch_size
        
        # 为训练集DataGenerator时，初始化steps_per_epoch
        if txt_path==train_txt_path:
            self.steps_per_epoch=self.data_num_all // self.batch_size
            global train_steps_per_epoch
            train_steps_per_epoch=self.steps_per_epoch
            #训练集按照给定的batch_size生成
            self.batch_size = batch_size
        elif txt_path == val_txt_path:
            #验证集的batch_size向训练集定下的steps_per_epoch对齐
            global val_steps_per_epoch
            self.batch_size = self.data_num_all // val_steps_per_epoch
            
        
        self.indexes_all=np.arange(self.data_num_all)
        self.to_fit = to_fit
        self.shuffle = shuffle
        self.on_epoch_end()
        self.input_size=input_size
        self.n_channels=3


    def __getitem__(self, index):
        """生成每一批次训练数据
        :param index: batch索引
        :return: 训练图像和标签
        """
        # 生成批次索引
        indexes_batch = self.indexes_all[index * self.batch_size:min((index + 1) * self.batch_size,self.data_num_all)]
        # 索引列表
        batch_path_list = [self.data_path_list[k] for k in indexes_batch]
        # 生成数据
        X = self._generate_X(batch_path_list)
        if self.to_fit:
            y = self._generate_y(batch_path_list)
            return X, y
        else:
            return X


    def __len__(self):
        """每个epoch下的批次数量
        """
        return int(np.floor(self.data_num_all / self.batch_size))


    def on_epoch_end(self):
        """每个epoch之后更新索引
        """
        self.indexes_all = np.arange(self.data_num_all)
        if self.shuffle == True:
            np.random.shuffle(self.indexes_all)


    def _generate_X(self, batch_path_list):
        """生成每一批次的图像
        :param batch_path_list: 批次数据索引列表
        :return: 一个批次的图像 shape(batch_size,w,h,channel)
        """
        # 初始化
        X = np.empty((self.batch_size, *self.input_size, self.n_channels))
        # 生成数据
        for i, path in enumerate(batch_path_list):
            # 存储一个批次
            X[i,] = self._load_h5(path[:-1],'x')
        return X


    def _generate_y(self, batch_path_list):
        """生成每一批次的图像
        :param batch_path_list: 批次数据索引列表
        :return: 一个批次的图像 shape(batch_size,w,h,channel)
        """
        # 初始化
        y = np.empty((self.batch_size, *self.input_size, 1))
        # 生成数据
        for i, path in enumerate(batch_path_list):
            # 存储一个批次
            y[i,] = self._load_h5(path[:-1],'y')
        return y


    def _load_h5(self, path, xory):
        """读取h5数据
        """
        h5_file = h5.File(path, "r")
        if(xory=='x'):
            np_file = h5_file.get('rgb')
            shape=np_file.shape
            #print(shape)
            #调整rgb通道至最后一维
            rgb_raw=np.transpose(np_file,(2,1,0))
            #降采样至224*224*3 标准输入
            rgb_downsample=cv2.resize(rgb_raw,dsize=(224, 224))
            return rgb_downsample
        elif(xory=='y'):
            np_file = h5_file.get('depth')
            shape=np_file.shape
            #print(shape)
            #调整深度图w,h
            depth_raw=np.transpose(np_file,(1,0))
            #降采样至224*224，增加维度至224*224*1标准输出 
            depth_downsample=cv2.resize(depth_raw,dsize=(224, 224))
            depth_downsample=np.expand_dims(depth_downsample,2)
            return depth_downsample
        else:
            print("error0")




class FastDepth:
    def __init__(self):
        self.build_net()

    def _SDWConv(self, filtres, kernel):
        def f(x):
            x = SeparableConv2D(filtres, kernel, padding='same')(x)
            x = BatchNormalization()(x)
            x = Activation('relu')(x)

            return x

        return f

    def _encoder(self):
        self.MN = MobileNet(input_shape=(224, 224, 3),
                            weights=None,
                            include_top='False')

        # 7*7*1024
        latent = self.MN.get_layer('conv_pw_13_relu').output

        return latent

    def _decoder(self, x):
        # 14*14*512
        x1 = self._SDWConv(512, (5, 5))(x)
        x1 = UpSampling2D()(x1)

        # 28*28*256
        x2 = self._SDWConv(256, (5, 5))(x1)
        x2 = UpSampling2D()(x2)
        s2 = self.MN.get_layer('conv_pw_5_relu').output
        x2 = add([x2, s2])

        # 56*56*128
        x3 = self._SDWConv(128, (5, 5))(x2)
        x3 = UpSampling2D()(x3)
        s3 = self.MN.get_layer('conv_pw_3_relu').output
        x3 = add([x3, s3])

        # 112*112*64
        x4 = self._SDWConv(64, (5, 5))(x3)
        x4 = UpSampling2D()(x4)
        s4 = self.MN.get_layer('conv_pw_1_relu').output
        x4 = add([x4, s4])

        # 224*224*32
        x5 = self._SDWConv(32, (5, 5))(x4)
        x5 = UpSampling2D()(x5)

        return x5

    def build_net(self):
        latent = self._encoder()
        out = self._decoder(latent)
        out_dense = Conv2D(1, (1, 1))(out)

        self.model = Model(inputs=self.MN.input, outputs=out_dense)


if __name__ == '__main__':


    # Generators
    # 72.76
    train_batchsize=64
    train_steps_per_epoch=10
    val_steps_per_epoch=20
    model_save_filepath="/data0/lijiaqi/smart_server/2019fastdepth/model_parameter_save"
    training_generator = DataGenerator(train_txt_path, to_fit=True,batch_size=train_batchsize)
    validation_generator = DataGenerator(val_txt_path, to_fit=True)
    net = FastDepth()
    #网络参数
    #net.model.summary()
    net.model.compile(optimizer=optimizers.SGD(lr=0.01, momentum=0.9, decay=0.0001, nesterov=False),
              loss='mse',
              metrics=['accuracy'])

    # checkpoint = ModelCheckpoint(model_save_filepath+'/'+datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),monitor='val_loss', save_weights_only=True,verbose=1,save_best_only=False, period=1)
    
    # if os.path.exists(filepath):
    #     model.load_weights(filepath)
    #     # 若成功加载前面保存的参数，输出下列信息
    #     print("checkpoint_loaded")
    
    print("compile finished")
    print("train_steps_per_epoch:",train_steps_per_epoch)
    print("validation_generator.batch_size:",validation_generator.batch_size)

    # test for DataGenerator
    # training_generator作为迭代器返回的turple结构为（迭代次数i，（train_x,train_y））
    # for i,(train_x,train_y) in enumerate(training_generator):
    #     print(train_x.shape)
    #     print(train_y.shape)
    #     print(i)
    #     break

    # train with fit_generator
    history=net.model.fit_generator(generator=training_generator,
                            validation_data=validation_generator,
                            epochs=100,
                            max_queue_size=64,
                            steps_per_epoch=train_steps_per_epoch,
                            validation_steps=val_steps_per_epoch,
                            callbacks=[ModelCheckpoint(os.path.join(model_save_filepath, 'model_{epoch:04d}_{val_loss:.6f}.hdf5'), 
                                                        monitor='val_loss', save_weights_only=True, 
                                                        verbose=1,save_best_only=False, period=1)],
                            verbose=1)
    accy=history.history['accuracy']
    lossy = history.history['loss']
    np_accy = np.array(accy).reshape((1,len(accy))) #reshape是为了能够跟别的信息组成矩阵一起存储
    np_lossy =np.array(lossy).reshape((1,len(lossy)))
    #np_out = np.concatenate([np_accy,np_lossy],axis=0)
    np.savetxt('history_acc.txt',np_accy)
    np.savetxt('history_loss.txt',np_lossy)       
    print("保存文件成功")

环境配置

数据集

 

 

 

 

Datagenerator

网络主体

Train

if __name__ == '__main__':
    # Generators
    # 72.76
    train_batchsize=64
    train_steps_per_epoch=10
    val_steps_per_epoch=20
    model_save_filepath="/data0/lijiaqi/smart_server/2019fastdepth/model_parameter_save"
    training_generator = DataGenerator(train_txt_path, to_fit=True,batch_size=train_batchsize)
    validation_generator = DataGenerator(val_txt_path, to_fit=True)
    net = FastDepth()
    #网络参数
    #net.model.summary()
    net.model.compile(optimizer=optimizers.SGD(lr=0.01, momentum=0.9, decay=0.0001, nesterov=False),
              loss='mse',
              metrics=['accuracy'])

    # checkpoint = ModelCheckpoint(model_save_filepath+'/'+datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),monitor='val_loss', save_weights_only=True,verbose=1,save_best_only=False, period=1)
    
    # if os.path.exists(filepath):
    #     model.load_weights(filepath)
    #     # 若成功加载前面保存的参数，输出下列信息
    #     print("checkpoint_loaded")
    
    print("compile finished")
    print("train_steps_per_epoch:",train_steps_per_epoch)
    print("validation_generator.batch_size:",validation_generator.batch_size)

    # test for DataGenerator
    # training_generator作为迭代器返回的turple结构为（迭代次数i，（train_x,train_y））
    # for i,(train_x,train_y) in enumerate(training_generator):
    #     print(train_x.shape)
    #     print(train_y.shape)
    #     print(i)
    #     break

    # train with fit_generator
    history=net.model.fit_generator(generator=training_generator,
                            validation_data=validation_generator,
                            epochs=100,
                            max_queue_size=64,
                            steps_per_epoch=train_steps_per_epoch,
                            validation_steps=val_steps_per_epoch,
                            callbacks=[ModelCheckpoint(os.path.join(model_save_filepath, 'model_{epoch:04d}_{val_loss:.6f}.hdf5'), 
                                                        monitor='val_loss', save_weights_only=True, 
                                                        verbose=1,save_best_only=False, period=1)],
                            verbose=1)
    accy=history.history['accuracy']
    lossy = history.history['loss']
    np_accy = np.array(accy).reshape((1,len(accy))) #reshape是为了能够跟别的信息组成矩阵一起存储
    np_lossy =np.array(lossy).reshape((1,len(lossy)))
    #np_out = np.concatenate([np_accy,np_lossy],axis=0)
    np.savetxt('history_acc.txt',np_accy)
    np.savetxt('history_loss.txt',np_lossy)       
    print("保存文件成功")

• compile部分的optimizers可以自定参数等等（参考官方中文文档，这一部分写得凑合（链接））

• 注意fit_generator需要自己定义steps_per_epoch，这个参数可以在datagenerator的init里对齐训练的batchsize修改steps_per_epoch全局变量，验证集的validation_steps也得定义，跟steps_per_epoch一样就行

• fit_generator利用callbacks机制保存每一个epoch的权重，改了一会，路径一般照着上面来就行

• verbose为0不输出epoch结果，为1输出进度条

• fit_generator返回一个history，照着上面保存accuracy和loss就行

其他注意事项

• 一般调代码训练的流程是：

• 写网络主体结构，compile之后summary看一下参数对不对

• 用几个random的数组作为traindata试一下输入输出或者中间结果的size符不符合预期

• 写datagenerator，生成enumerate打印每次的生成数据对不对

• 套进去开训，中间参数checkpoints啥的细节优化