在深度学习和机器学习的实践中，数据集的大小常常会对模型的训练效果和计算资源产生重大影响。尤其是在数据集过大的情况下，存储、处理和训练时间都会显著增加。有些时候我们拿到了庞大的数据集，而并不需要用到那么多数据集进行训练，为了提高模型的效率和准确性，需要对数据集进行优化，随机删除部分不必要的文件，可以有效减小数据集规模，同时保留我们需要的信息。
本文代码即是针对数据进行随机对应删除操作，即给定文件名中包含的字符及删除百分比，运行代码后可以自动删除同名的图片及标注数据，十分方便。删除有风险，建议删除之前做好文件备份。
以下代码为函数定义，输入图片、标注文件夹路径及删除百分比，可以做到随机删除。
def delete_files(folder_jpg, folder_xml, delete_percentage):
jpg_files = os.listdir(folder_jpg)
xml_files = os.listdir(folder_xml)
jpg_names = [os.path.splitext(f)[0] for f in jpg_files]
xml_names = [os.path.splitext(f)[0] for f in xml_files]
jpg_names.sort()
xml_names.sort()
num_files_to_delete = int(len(jpg_names) * (delete_percentage / 100))
files_to_delete = random.sample(range(len(jpg_names)), num_files_to_delete)
for idx in files_to_delete:
jpg_name = jpg_names[idx]
xml_name = xml_names[idx]
jpg_file = os.path.join(folder_jpg, jpg_name + ".jpg")
xml_file = os.path.join(folder_xml, xml_name + ".xml")
if os.path.exists(jpg_file):
os.remove(jpg_file)
print(f"Deleted {jpg_file}")
if os.path.exists(xml_file):
os.remove(xml_file)
print(f"Deleted {xml_file}")
如果仅想删除文件名中包含指定字符的，可以使用以下函数定义
def delete_files_with_string(folder_jpg, folder_xml, string_to_find, delete_percentage):
jpg_files = os.listdir(folder_jpg)
xml_files = os.listdir(folder_xml)
jpg_names = [os.path.splitext(f)[0] for f in jpg_files]
xml_names = [os.path.splitext(f)[0] for f in xml_files]
jpg_names.sort()
xml_names.sort()
num_files_to_delete = int(len(jpg_names) * (delete_percentage / 100))
files_to_delete = random.sample(range(len(jpg_names)), num_files_to_delete)
for idx in files_to_delete:
jpg_name = jpg_names[idx]
xml_name = xml_names[idx]
if string_to_find in jpg_name:
jpg_file = os.path.join(folder_jpg, jpg_name + ".jpg")
xml_file = os.path.join(folder_xml, xml_name + ".xml")
if os.path.exists(jpg_file):
os.remove(jpg_file)
print(f"Deleted {jpg_file}")
if os.path.exists(xml_file):
os.remove(xml_file)
print(f"Deleted {xml_file}")
这里默认图片文件后缀为jpg，标注文件后缀为xml，如果遇到文件后缀名为大写的，则需要先将大写调整为小写，再进行查找，使用以下代码进行后缀名大小改小写
def upp2low(directory):
converted_count = 0
if not os.path.exists(directory):
raise FileNotFoundError(f"Directory {directory} does not exist.")
for filename in os.listdir(directory):
file_path = os.path.join(directory, filename)
if os.path.isfile(file_path):
name, extension = os.path.splitext(filename)
if extension.isupper():
new_filename = name + extension.lower()
new_file_path = os.path.join(directory, new_filename)
os.rename(file_path, new_file_path)
converted_count += 1
print(f"Renamed: {filename} -> {new_filename}")
print(f"All file suffixes in the folder are lowercase, and a total of {converted_count} files have been processed")
return converted_count
这样就可以保证所有相同格式的图片及标注都可以被准确统计并部分随机删除。
以下为完整代码
import os
import random
import shutil
def upp2low(directory):
converted_count = 0
# 检查目录是否存在
if not os.path.exists(directory):
raise FileNotFoundError(f"Directory {directory} does not exist.")
# 遍历文件夹中的所有文件
for filename in os.listdir(directory):
file_path = os.path.join(directory, filename)
if os.path.isfile(file_path):
name, extension = os.path.splitext(filename)
if extension.isupper():
new_filename = name + extension.lower()
new_file_path = os.path.join(directory, new_filename)
os.rename(file_path, new_file_path)
converted_count += 1
print(f"Renamed: {filename} -> {new_filename}")
print(f"All file suffixes in the folder are lowercase, and a total of {converted_count} files have been processed")
return converted_count
def delete_files_with_string(folder_jpg, folder_xml, string_to_find, delete_percentage):
jpg_files = os.listdir(folder_jpg)
xml_files = os.listdir(folder_xml)
jpg_names = [os.path.splitext(f)[0] for f in jpg_files]
xml_names = [os.path.splitext(f)[0] for f in xml_files]
jpg_names.sort()
xml_names.sort()
num_files_to_delete = int(len(jpg_names) * (delete_percentage / 100))
files_to_delete = random.sample(range(len(jpg_names)), num_files_to_delete)
for idx in files_to_delete:
jpg_name = jpg_names[idx]
xml_name = xml_names[idx]
if string_to_find in jpg_name:
jpg_file = os.path.join(folder_jpg, jpg_name + ".jpg") #文件后缀名
xml_file = os.path.join(folder_xml, xml_name + ".xml")
if os.path.exists(jpg_file):
os.remove(jpg_file)
print(f"Deleted {jpg_file}")
if os.path.exists(xml_file):
os.remove(xml_file)
print(f"Deleted {xml_file}")
if __name__ == "__main__":
jpg_folder = r" " #修改图片文件夹
xml_folder = r" " #修改标签文件夹
delete_percentage = 10 # 删除百分之10的含有指定字符串的图片及其对应的XML文件
string_to_find = 'B' #设置字符
upp2low(jpg_folder)
upp2low(xml_folder)
# 调用函数删除文件
delete_files_with_string(jpg_folder, xml_folder,string_to_find, delete_percentage)
也可以先随机生成部分对应文件，再删除测试效果。
import os
import random
import string
def generate_files_with_string(folder1, folder2, string_to_include, num_files, file_extension1, file_extension2):
for i in range(num_files):
file_name = ''.join(random.choices(string.ascii_letters + string.digits, k=10))
file_name_with_string = file_name + "_" + string_to_include
file_path1 = os.path.join(folder1, file_name_with_string + "." + file_extension1)
file_path2 = os.path.join(folder2, file_name_with_string + "." + file_extension2)
with open(file_path1, 'w') as f1, open(file_path2, 'w') as f2:
f1.write("This file contains the specified string: " + string_to_include)
f2.write("This is another file associated with the first one.")
print(f"Generated files: {file_path1}, {file_path2}")
if __name__ == "__main__":
# 设置文件夹路径和需要生成的文件属性
folder1 = r" " # 第一个文件夹路径
folder2 = r" " # 第二个文件夹路径
string_to_include = "B" # 指定字符串
num_files = 10 # 需要生成的文件数量
file_extension1 = "jpg" # 第一个文件后缀名
file_extension2 = "xml" # 第二个文件后缀名
generate_files_with_string(folder1, folder2, string_to_include, num_files, file_extension1, file_extension2)
扫一扫
312
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
