【14】模型训练自制数据集前的一些数据处理操作

坏图像扫描与检查

import os
import cv2

# 检查的图像数据和保存的路径设置一样，则会检测出坏图像
dataDir = r"/home/s22030812007/faster-rcnn-pytorch/VOCdevkit1/VOC2007/JPEGImages/"
saveDir = r"/home/s22030812007/faster-rcnn-pytorch/VOCdevkit1/VOC2007/JPEGImages/"

if not os.path.exists(saveDir):
    os.makedirs(saveDir)
c = 0
for one_pic in os.listdir(dataDir):
    one_path = dataDir + one_pic
    one_img = cv2.imread(one_path)
    new_path = saveDir + one_pic
    d = cv2.imwrite(new_path, one_img)
    c = c + 1
    print(d, c)

所有文件连续重命名排号

import os
if __name__ == '__main__':
    path = r"C:/Users/27801/Documents/Datasets/5551"  # 需要出来的图片文件所在路径(即文件夹路径)，注意斜杠的方向
    file_names = os.listdir(path)  # 获取该文件夹下所有的文件名称（包括文件夹）
    print(f'file_names:{file_names}')
    count = 6563  # 设置变量count，为重命名文件的起始编号
    for file in file_names:  # 遍历所有文件名
        print(f"file:{file}")
        old_name = os.path.join(path, file)
        print(f'old_name:{old_name}')
        if os.path.isdir(old_name):  # 如果是文件夹则跳过
            continue
        filename = os.path.splitext(file)[0]  # 文件名
        filetype = os.path.splitext(file)[1]  # 文件扩展名
        new_name = os.path.join(path, str(count).zfill(6) + filetype)  # 用字符串函数zfill 以0补全所需位数
        print(f'new_name:{new_name}')
        os.rename(old_name, new_name)  # 重命名 用新的文件名代替旧的文件名
        count += 1  # count的作用是给文件名计数的，以及累积的步幅

划分数据集为训练集、测试集和验证集

import os, shutil, random

random.seed(201)
import numpy as np
from sklearn.model_selection import train_test_split

# 验证集、测试集的占比，剩余的比例为训练集数据，0.1为占比百分之十
val_size = 0.1
test_size = 0.1
postfix = 'jpg'

# 图像数据的路径
imgpath = r'C:/Users/27801/Desktop/JPEGImages/'

#txt后缀的文件路径
txtpath = r'C:/Users/27801/Desktop/txts/'

#创建目录文件夹，用于存放划分数据后的图像，且若不存在则自动创建文件夹
os.makedirs('Logos3/images/train', exist_ok=True)
os.makedirs('Logos3/images/val', exist_ok=True)
os.makedirs('Logos3/images/test', exist_ok=True)
os.makedirs('Logos3/labels/train', exist_ok=True)
os.makedirs('Logos3/labels/val', exist_ok=True)
os.makedirs('Logos3/labels/test', exist_ok=True)

# 列出txtpath目录下所有包含'txt'字符串的文件名  将这些文件名存储在一个列表中
listdir = np.array([i for i in os.listdir(txtpath) if 'txt' in i])

# 随机打乱列表中文件名
random.shuffle(listdir)
# 按照比例划分
train, val, test = listdir[:int(len(listdir) * (1 - val_size - test_size))], \
    listdir[int(len(listdir) * (1 - val_size - test_size)):int(len(listdir) * (1 - test_size))], \
    listdir[int(len(listdir) * (1 - test_size)):]
print(f'train set size:{len(train)} val set size:{len(val)} test set size:{len(test)}')

# 将分配好的文件进行转移到指定文件夹下
for i in train:
    shutil.copy('{}/{}.{}'.format(imgpath, i[:-4], postfix), 'Logos3/images/train/{}.{}'.format(i[:-4], postfix))
    shutil.copy('{}/{}'.format(txtpath, i), 'Logos3/labels/train/{}'.format(i))

for i in val:
    shutil.copy('{}/{}.{}'.format(imgpath, i[:-4], postfix), 'Logos3/images/val/{}.{}'.format(i[:-4], postfix))
    shutil.copy('{}/{}'.format(txtpath, i), 'Logos3/labels/val/{}'.format(i))

for i in test:
    shutil.copy('{}/{}.{}'.format(imgpath, i[:-4], postfix), 'Logos3/images/test/{}.{}'.format(i[:-4], postfix))
    shutil.copy('{}/{}'.format(txtpath, i), 'Logos3/labels/test/{}'.format(i))

将标注的json文件转换成yolo格式文件，即txt文件

import json
import os

## json格式转换成yolo格式
# 设置：标签对应的编号
# ！！！一定要查看自己标注数据集时对应的编号
name2id = {'figure': 0, 'text': 1, 'mix': 2}

def convert(img_size, box):
    dw = 1. / (img_size[0])
    dh = 1. / (img_size[1])
    x = (box[0] + box[2]) / 2.0 - 1
    y = (box[1] + box[3]) / 2.0 - 1
    w = box[2] - box[0]
    h = box[3] - box[1]
    x = x * dw
    w = w * dw
    y = y * dh
    h = h * dh
    return (x, y, w, h)


def decode_json(json_folder_path, json_name):

    # 设置：转换好格式后的标签存放的路径
    txt_name = 'C:/Users/27801/Documents/Datasets/mydata/txt/' + json_name[0:-5] + '.txt'
    txt_file = open(txt_name, 'w')

    json_path = os.path.join(json_folder_path, json_name)
    data = json.load(open(json_path, 'r', encoding='gb2312'))

    img_w = data['imageWidth']
    img_h = data['imageHeight']

    for i in data['shapes']:
        label_name = i['label']
        if (i['shape_type'] == 'rectangle'):
            x1 = int(i['points'][0][0])
            y1 = int(i['points'][0][1])
            x2 = int(i['points'][1][0])
            y2 = int(i['points'][1][1])

            bb = (x1, y1, x2, y2)
            bbox = convert((img_w, img_h), bb)
            txt_file.write(str(name2id[label_name]) + " " + " ".join([str(a) for a in bbox]) + '\n')

if __name__ == "__main__":

    # 路径设置：需要进行格式转换的.json文件夹存放路径
    json_folder_path = 'C:/Users/27801/Documents/Datasets/mydata/labels'
    json_names = os.listdir(json_folder_path)
    for json_name in json_names:
        decode_json(json_folder_path, json_name)

将xml格式文件转换成yolo格式可读取的文件

import cv2
import os
import xml.etree.ElementTree as ET
import numpy as np

classes = ['text', 'mix', 'figure']

def convert(size, box):
    dw = 1. / (size[0])
    dh = 1. / (size[1])
    x = (box[0] + box[1]) / 2.0 - 1
    y = (box[2] + box[3]) / 2.0 - 1
    w = box[1] - box[0]
    h = box[3] - box[2]
    x = x * dw
    w = w * dw
    y = y * dh
    h = h * dh
    return (x, y, w, h)

def convert_annotation(xmlpath, xmlname):
    with open(xmlpath, "r", encoding='utf-8') as in_file:
        txtname = xmlname[:-4] + '.txt'
        txtfile = os.path.join(txtpath, txtname)
        tree = ET.parse(in_file)
        root = tree.getroot()
        filename = root.find('filename')
        img = cv2.imdecode(np.fromfile('{}/{}.{}'.format(imgpath, xmlname[:-4], postfix), np.uint8), cv2.IMREAD_COLOR)
        h, w = img.shape[:2]
        res = []
        for obj in root.iter('object'):
            cls = obj.find('name').text
            if cls not in classes:
                classes.append(cls)
            cls_id = classes.index(cls)
            xmlbox = obj.find('bndbox')
            b = (float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text),
                 float(xmlbox.find('ymax').text))
            bb = convert((w, h), b)
            res.append(str(cls_id) + " " + " ".join([str(a) for a in bb]))
        if len(res) != 0:
            with open(txtfile, 'w+') as f:
                f.write('\n'.join(res))

if __name__ == "__main__":
    # 图像后缀为jpg，图像文件路径、xml标注文件路径
    postfix = 'jpg'
    imgpath = r'C:/Users/27801/Desktop/JPEGImages/'
    xmlpath = r'C:/Users/27801/Desktop/Annotations/'

    # txt为yolo的表现格式，若没有存在，则自动创建文件夹
    txtpath = r'C:/Users/27801/Desktop/txts/'

    if not os.path.exists(txtpath):
        os.makedirs(txtpath, exist_ok=True)

    list = os.listdir(xmlpath)
    error_file_list = []
    for i in range(0, len(list)):
        try:
            path = os.path.join(xmlpath, list[i])
            if ('.xml' in path) or ('.XML' in path):
                convert_annotation(path, list[i])
                print(f'file {list[i]} convert success.')
            else:
                print(f'file {list[i]} is not xml format.')
        except Exception as e:
            print(f'file {list[i]} convert error.')
            print(f'error message:\n{e}')
            error_file_list.append(list[i])
    print(f'this file convert failure\n{error_file_list}')
    print(f'Dataset Classes:{classes}')

将gt后缀的图像数据文件进行解析

if __name__ == "__main__":
    # 下载的数据为gt格式存放
    label_path = r'C:\Users\27801\Desktop\BelgaLogos\qset3_internal_and_local.gt'
    labels_origin = []
    with open(label_path, "r") as f:
        for line in f.readlines():
            l = " ".join(line.split("\t"))
            labels_origin.append(l.strip("\n"))
    imgs_label = {}
    # 数据包括的所有标签
    classes = [['Adidas', 'Adidas-text', 'Airness', 'Base', 'BFGoodrich', 'Bik', 'Bouigues', 'Bridgestone',
                'Bridgestone-text', 'Carglass', 'Citroen', 'Citroen-text', 'CocaCola', 'Cofidis', 'Dexia',
                'ELeclerc', 'Ferrari', 'Gucci', 'Kia', 'Mercedes', 'Nike', 'Peugeot', 'Puma', 'Puma-text',
                'Quick', 'Reebok', 'Roche', 'Shell', 'SNCF', 'Standard_Liege', 'StellaArtois', 'TNT',
                'Total', 'Umbro', 'US_President', 'Veolia', 'VRT']]
    for i in labels_origin:
        if i.split()[2] not in imgs_label.keys():
            imgs_label[i.split()[2]] = []
        if i.split()[1] not in classes:
            classes.append(i.split()[1])
        imgs_label[i.split()[2]].append(
            [int(i.split()[5]), int(i.split()[6]), int(i.split()[7]), int(i.split()[8]), i.split()[1]])
    print('imgs_label', imgs_label)

统计yolo存储文件下各类标签下所对应标注目标的数量，累积打印数据分布情况

import matplotlib.pyplot as plt
import os

plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False

cn_path = open(r"C:\Users\27801\Desktop\BelgaLogos\classes.txt")  # 存有类别的文本路径，如："C:/Users/Admin/Desktop/classes.txt"

classes = [i.replace("\n", "") for i in cn_path.readlines()]
print(classes)

class_dict = {i: 0 for i in classes}
print("类别数", len(class_dict))

def main(base_path):
    # 列出指定路径base_path下的所有文件和目录名
    fileList = os.listdir(base_path)
    for file in fileList:
        if file == "classes.txt":
            continue
        with open(base_path + file, 'r') as f:
            for i in f.readlines():
                i = i.split(' ')  # 按照空格进行划分文件内数值，首字符代表标签类别
                class_dict[classes[int(i[0])]] += 1
    fig, ax = plt.subplots(figsize=(10, 8))
    plt.title('数量')
    plt.xticks(rotation=90)  # x轴文字方向旋转90度
    bars = plt.bar(class_dict.keys(), class_dict.values())
    # 绘制柱形图部分
    for b in bars:
        height = b.get_height()
        ax.annotate(f'{height}',
                    # xy控制的是，标注哪个点，x=x坐标+width/2, y=height，即柱子上平面的中间
                    xy=(b.get_x() + b.get_width() / 2, height),
                    xytext=(0, 3),  # 文本放置的位置，如果有textcoords，则表示是针对xy位置的偏移，否则是图中的固定位置
                    textcoords="offset points",  # 两个选项 'offset pixels'，'offset pixels'
                    va='bottom', ha='center'  # 代表verticalalignment 和horizontalalignment，控制水平对齐和垂直对齐。
                    )
    plt.savefig('./统计.png',  # ⽂件名：png、jpg、pdf
                dpi=100,  # 保存图⽚像素密度
                bbox_inches='tight')  # 保存图⽚完整
    plt.show()

if __name__ == '__main__':
    # 存放txt文件的路径，末尾的反斜杠不可少
    base_path = r"C:/Users/27801/Desktop/BelgaLogos/txts/"
    main(base_path)

如果你觉得这篇文章对你有所启发的话，期待你的点赞、收藏和打赏，对我创作的支持！！！