Dataset: add READMEs, organize scripts, update .gitignore

- Add README.md to all 16 dataset directories with script references
- Centralize generation scripts under scripts/ with full README
- Update .gitignore to cover all data directories, keep only metadata
- Remove redundant CCPD/.gitignore (parent covers same rules)
- Remove stale scripts/test/ (254MB pipeline residue)
- Fix label parser reference: scripts are authoritative source of truth
This commit is contained in:
kirl committed 2026-06-23 14:41:39 +08:00
1 parent 7867c3bfaf
commit a64690477a
43 files changed
+2691 -205

No files matched your search

+2
View File
@@ -0,0 +1,2 @@
test/
train/
+69
View File
@@ -0,0 +1,69 @@
# CCPD_19K_EXP_CUT — 拓展裁剪 + 角点重映射
> **以脚本为准**:生成逻辑以 `ICE_Dataset/scripts/build_ccpd_19k_exp_cut.py` 为准。
由 CCPD_19K 经 YOLOv5t 推理 → bbox 拓展 → 裁剪 + 角点重映射生成。
## 处理流程
```
CCPD_19K (19,000 张)
→ YOLOv5t best.pt 推理 (imgsz=352, conf=0.25)
→ bbox 拓展:
常规: 12% (w ≥ 67 且 h ≥ 51)
小车牌: 18% (w < 67 或 h < 51)
→ 裁剪
→ 角点重映射写入文件名
→ CCPD_19K_EXP_CUT
```
## 目录结构
```
CCPD_19K_EXP_CUT/
├── train/ # 16,999 张
│ ├── blue/ # 4,435 张
│ ├── green/ # 4,486 张
│ ├── police/ # 897 张
│ ├── army/ # 898 张
│ ├── gangao/ # 899 张
│ ├── shiguan_old/ # 898 张
│ ├── shiguan_new/ # 897 张
│ ├── lingguan_old/ # 897 张
│ ├── lingguan_new/ # 897 张
│ ├── coach/ # 897 张
│ └── new_energy/ # 898 张
├── test/ # 1,888 张
│ └── ... (同上 11 类)
└── README.md
```
## 统计
| 指标 | 数值 |
|------|------|
| 源图 | 19,000 |
| 裁剪成功 | 18,887 (99.4%) |
| 角点重映射 | 18,870 (99.3%) |
| 跳过 | 113 (0.6%) |
| - 漏检 | 71 |
| - 多框 | 42 |
## 文件名角点格式
裁剪后 CCPD 文件名中的角点段已更新为裁剪坐标系像素坐标:
```
原: ...-324&452_241&449_242&420_325&423-...
新: ...-99&38_16&35_17&6_100&9-...
```
格式: `-C1X&C1Y_C2X&C2Y_C3X&C3Y_C4X&C4Y-` (像素坐标, 整数)
## 生成脚本
```bash
cd ICE_Dataset/CCPD_19K_EXP_CUT
python build_ccpd_19k_exp_cut.py --dry-run # 预览
python build_ccpd_19k_exp_cut.py --full # 全量
```
+422
View File
@@ -0,0 +1,422 @@
#!/usr/bin/env python3
"""
CCPD_19K 模型推理 → bbox 拓展 → 裁剪 → 角点重映射写入文件名 → CCPD_19K_EXP_CUT
使用 YOLOv5t best.pt 对 CCPD_19K 全部图片进行推理,
按 12%/18% 策略拓展 bbox 后裁剪,并将原 CCPD 角点重映射到裁剪坐标系后更新文件名。
保持原文件夹结构,无额外标签文件。
用法:
python build_ccpd_19k_exp_cut.py --dry-run # 预览 (每类2张, 图上绘制角点)
python build_ccpd_19k_exp_cut.py --n 100 # 预览 N 张
python build_ccpd_19k_exp_cut.py --full # 全量处理 (~19K, 无绘制)
"""
import argparse
import os
import re
import shutil
import sys
import time
from collections import defaultdict
from pathlib import Path
import cv2
import numpy as np
# ---- Unicode-safe I/O (must be BEFORE ultralytics import!) ----
_original_imread = cv2.imread
def _imread_unicode(filepath, flags=cv2.IMREAD_COLOR):
try:
data = np.fromfile(filepath, dtype=np.uint8)
if len(data) == 0:
return None
img = cv2.imdecode(data, flags)
return img
except Exception:
return _original_imread(filepath, flags)
def _imwrite_unicode(filepath, img, params=None):
try:
ext = os.path.splitext(filepath)[1]
ok, buf = cv2.imencode(ext, img, params or [])
if ok:
with open(filepath, 'wb') as f:
f.write(buf.tobytes())
return True
return False
except Exception:
return cv2.imwrite(filepath, img, params)
cv2.imread = _imread_unicode # patch before ultralytics hooks it
# -----------------------------------------------------------------
import torch
import yaml
from ultralytics import YOLO
# ---------------------------------------------------------------------------
# 路径
# ---------------------------------------------------------------------------
SCRIPT_DIR = Path(__file__).resolve().parent
PROJ_ROOT = SCRIPT_DIR.parents[1] # ICEv2/
MODEL_PATH = PROJ_ROOT / "ICE_NNv2" / "yolov5t" / "output" / "1_CCPD_19K" / "weights" / "best.pt"
SRC_DIR = PROJ_ROOT / "ICE_Dataset" / "CCPD_19K" # 源数据集
DST_DIR = SCRIPT_DIR # 本目录 (CCPD_19K_EXP_CUT)
# ---------------------------------------------------------------------------
# 推理 & 拓展参数 (与 ICE 主项目 / infer_expand.py 一致)
# ---------------------------------------------------------------------------
IMGSZ = 352
CONF = 0.25
# CCPD 图像原始分辨率
CCPD_W = 720
CCPD_H = 1160
# ICE 主项目 bbox 拓展系数: 12% 常规, 18% 小车牌
EXPAND_NORMAL = 1.12
EXPAND_SMALL = 1.18
# ICE: bw < 180 || bh < 48 @1080p → 缩放到 CCPD 分辨率
SMALL_BOX_W = int(180 * CCPD_W / 1920) # ≈ 68
SMALL_BOX_H = int(48 * CCPD_H / 1080) # ≈ 52
MIN_PAD = 2
# CCPD_19K 的 split 名称
SPLITS = ["train", "test"]
# ---------------------------------------------------------------------------
# CCPD 角点解析 & 文件名重映射
# ---------------------------------------------------------------------------
# 匹配 CCPD 文件名中角点段: -C1X&C1Y_C2X&C2Y_C3X&C3Y_C4X&C4Y-
CORNER_RE = re.compile(r'-(\d+)&(\d+)_(\d+)&(\d+)_(\d+)&(\d+)_(\d+)&(\d+)-')
def parse_ccpd_corners(filename: str):
"""
从 CCPD 文件名解析 4 个角点 (四边形), 返回像素坐标.
格式: ...-C1X&C1Y_C2X&C2Y_C3X&C3Y_C4X&C4Y-...
返回: [(x1,y1), (x2,y2), (x3,y3), (x4,y4)] 或 None
"""
m = CORNER_RE.search(filename)
if not m:
return None
nums = [int(m.group(i)) for i in range(1, 9)]
return [(nums[0], nums[1]), (nums[2], nums[3]), (nums[4], nums[5]), (nums[6], nums[7])]
def remap_corners_to_crop(corners, x1, y1, x2, y2):
"""
将角点从原图坐标系重映射到裁剪区域像素坐标 (整数).
仅当四边形与裁剪区重叠 > 25% 时才重映射.
返回: [(px,py), ...] 裁剪区像素坐标 或 None
"""
crop_w = x2 - x1
crop_h = y2 - y1
if crop_w <= 0 or crop_h <= 0:
return None
# 四边形外包矩形与裁剪区重叠率
xs = [c[0] for c in corners]
ys = [c[1] for c in corners]
poly_x1, poly_x2 = min(xs), max(xs)
poly_y1, poly_y2 = min(ys), max(ys)
inter_w = max(0, min(poly_x2, x2) - max(poly_x1, x1))
inter_h = max(0, min(poly_y2, y2) - max(poly_y1, y1))
poly_area = (poly_x2 - poly_x1) * (poly_y2 - poly_y1)
if poly_area <= 0 or (inter_w * inter_h) / poly_area <= 0.25:
return None
# 逐点平移 + clip
new_corners = []
for (px, py) in corners:
nx = max(0, min(crop_w - 1, int(round(px - x1))))
ny = max(0, min(crop_h - 1, int(round(py - y1))))
new_corners.append((nx, ny))
return new_corners
def replace_corners_in_filename(filename: str, new_corners):
"""
替换文件名中的角点段为新的像素坐标.
new_corners: [(px1,py1), (px2,py2), (px3,py3), (px4,py4)]
"""
new_seg = "-{0}&{1}_{2}&{3}_{4}&{5}_{6}&{7}-".format(
new_corners[0][0], new_corners[0][1],
new_corners[1][0], new_corners[1][1],
new_corners[2][0], new_corners[2][1],
new_corners[3][0], new_corners[3][1],
)
return CORNER_RE.sub(new_seg, filename, count=1)
# ---------------------------------------------------------------------------
# 单张处理
# ---------------------------------------------------------------------------
def infer_and_crop(model, img_path: Path, out_img_path: Path, draw_corners: bool = False):
"""
对单张图片:
1. YOLO 推理 → 检出框
2. 按大小选择 12%/18% 拓展
3. 裁剪, 角点重映射 → 更新文件名
4. (可选) 在裁剪图上绘制四边形角点供审核
返回:
"cropped" — 成功 (角点已重映射写入文件名)
"cropped_nocorners" — 成功 (源文件名无角点段, 原名保存)
"cropped_low_overlap" — 成功 (角点重叠不足, 原名保存)
"miss" — 漏检
"multi" — 多框跳过
"""
try:
with open(str(img_path), 'rb') as f:
data = f.read()
img = cv2.imdecode(np.frombuffer(data, np.uint8), cv2.IMREAD_COLOR)
except Exception:
img = None
if img is None:
return "miss"
orig_h, orig_w = img.shape[:2]
results = model.predict(
str(img_path), imgsz=IMGSZ, verbose=False,
conf=CONF, max_det=5
)
boxes = results[0].boxes
if boxes is None or len(boxes) == 0:
return "miss"
xywh = boxes.xywh.cpu().numpy()
if len(xywh) > 1:
return "multi"
cx, cy, bw, bh = xywh[0]
# ---- 拓展系数 ----
expand_factor = EXPAND_SMALL if (bw < SMALL_BOX_W or bh < SMALL_BOX_H) else EXPAND_NORMAL
# ---- 裁剪区域 ----
pad_x = max(MIN_PAD, int(round(bw * (expand_factor - 1.0) / 2.0)))
pad_y = max(MIN_PAD, int(round(bh * (expand_factor - 1.0) / 2.0)))
x1_raw = int(cx - bw / 2)
y1_raw = int(cy - bh / 2)
x2_raw = int(cx + bw / 2)
y2_raw = int(cy + bh / 2)
x1 = max(x1_raw - pad_x, 0)
y1 = max(y1_raw - pad_y, 0)
x2 = min(x2_raw + pad_x + 1, orig_w)
y2 = min(y2_raw + pad_y + 1, orig_h)
if x2 - x1 <= 0 or y2 - y1 <= 0:
return "miss"
crop = img[y1:y2, x1:x2]
# ---- 角点重映射 → 更新文件名 ----
stem = img_path.stem
corners = parse_ccpd_corners(stem)
remapped = None
if corners is not None:
remapped = remap_corners_to_crop(corners, x1, y1, x2, y2)
if remapped is not None:
new_stem = replace_corners_in_filename(stem, remapped)
else:
new_stem = stem # 重叠不足, 保留原名
else:
new_stem = stem # 无角点段
# ---- 绘制角点 (预览模式) ----
if draw_corners and remapped is not None:
crop_h_draw, crop_w_draw = crop.shape[:2]
pts = [(px, py) for (px, py) in remapped]
pts_arr = np.array(pts, dtype=np.int32)
cv2.polylines(crop, [pts_arr], isClosed=True, color=(0, 255, 0), thickness=2)
for k, (px, py) in enumerate(pts):
cv2.circle(crop, (px, py), 3, (0, 0, 255), -1)
cv2.putText(crop, str(k), (px + 4, py - 4),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1)
label = f"expand={expand_factor:.0%}"
cv2.putText(crop, label, (4, 14),
cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 255, 255), 1)
# ---- 保存 ----
out_img_path.parent.mkdir(parents=True, exist_ok=True)
final_out = out_img_path.parent / (new_stem + out_img_path.suffix)
_imwrite_unicode(str(final_out), crop)
if corners is None:
return "cropped_nocorners"
elif remapped is None:
return "cropped_low_overlap"
else:
return "cropped"
# ---------------------------------------------------------------------------
# 主流程
# ---------------------------------------------------------------------------
def main():
parser = argparse.ArgumentParser(description="CCPD_19K → 推理拓展裁剪+角点重映射 → CCPD_19K_EXP_CUT")
parser.add_argument("--dry-run", action="store_true", help="预览 (每类2张, 图上绘制角点)")
parser.add_argument("--n", type=int, default=None, help="预览 N 张")
parser.add_argument("--full", action="store_true", help="全量处理 (~19K)")
args = parser.parse_args()
if not (args.dry_run or args.n or args.full):
print("请指定 --dry-run, --n N, 或 --full")
print(f" python {Path(__file__).name} --dry-run")
print(f" python {Path(__file__).name} --full")
return
if args.dry_run:
per_class_limit = 2
label = "DRY-RUN (每类2张, 绘制角点)"
elif args.n:
per_class_limit = None
label = f"预览 ({args.n} 张)"
else:
per_class_limit = None
label = "全量处理"
draw = not args.full # 非全量模式绘制角点
print("=" * 70)
print(f"CCPD_19K → bbox拓展 {EXPAND_NORMAL:.0%}/{EXPAND_SMALL:.0%} → 裁剪 → 角点重映射写入文件名")
print(f"模式: {label}")
print("=" * 70)
# ---- 加载模型 ----
print(f"\n加载: {MODEL_PATH}")
device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
model = YOLO(str(MODEL_PATH))
model.to(device)
print(f"设备: {device} | imgsz={IMGSZ} | conf={CONF}")
print(f"拓展: 常规 {EXPAND_NORMAL:.0%} (w>={SMALL_BOX_W},h>={SMALL_BOX_H}) / 小车牌 {EXPAND_SMALL:.0%} (w<{SMALL_BOX_W}或h<{SMALL_BOX_H})")
if draw:
print(f"预览: 裁剪图上绘制四边形角点 (绿框+红顶点+序号)")
# ---- 收集图片 ----
print("\n收集图片...")
all_tasks = [] # [(src_path, out_path)]
for split in SPLITS:
split_dir = SRC_DIR / split
if not split_dir.exists():
continue
for cls_dir in sorted(split_dir.iterdir()):
if not cls_dir.is_dir():
continue
for jpg in sorted(cls_dir.glob("*.jpg")):
rel = jpg.relative_to(SRC_DIR)
out = DST_DIR / rel
all_tasks.append((jpg, out))
print(f"共 {len(all_tasks)} 张")
# ---- 限制数量 ----
if args.dry_run:
by_class = defaultdict(list)
for src, dst in all_tasks:
by_class[src.parent.name].append((src, dst))
limited = []
for cls in sorted(by_class):
limited.extend(by_class[cls][:per_class_limit])
all_tasks = limited
print(f"预览: 限制 {len(all_tasks)} 张 (每类 {per_class_limit} 张)")
elif args.n:
all_tasks = all_tasks[:args.n]
print(f"预览: 限制 {len(all_tasks)} 张")
# ---- 清理旧输出 (非 full) ----
if not args.full and DST_DIR.exists():
shutil.rmtree(DST_DIR)
# ---- 处理 ----
print(f"\n开始处理...")
t0 = time.perf_counter()
stats = {"total": 0, "cropped": 0, "nocorners": 0, "low_overlap": 0, "miss": 0, "multi": 0}
sc = defaultdict(lambda: {"ok": 0, "skip": 0})
for idx, (src_path, out_path) in enumerate(all_tasks):
rel = src_path.relative_to(SRC_DIR)
parts = rel.parts # (split, class, filename)
split_name, cls_name = parts[0], parts[1]
result = infer_and_crop(model, src_path, out_path, draw_corners=draw)
stats["total"] += 1
key = f"{split_name}/{cls_name}"
if result == "cropped":
stats["cropped"] += 1
sc[key]["ok"] += 1
elif result == "cropped_nocorners":
stats["nocorners"] += 1
sc[key]["ok"] += 1
elif result == "cropped_low_overlap":
stats["low_overlap"] += 1
sc[key]["ok"] += 1
elif result in ("miss", "multi"):
stats[result] += 1
sc[key]["skip"] += 1
if (idx + 1) % 500 == 0 or (idx + 1) == len(all_tasks):
elapsed = time.perf_counter() - t0
rate = (idx + 1) / elapsed if elapsed > 0 else 0
eta = (len(all_tasks) - (idx + 1)) / rate if rate > 0 else 0
ok = stats["cropped"] + stats["nocorners"] + stats["low_overlap"]
skip = stats["miss"] + stats["multi"]
print(f" [{idx+1}/{len(all_tasks)}] "
f"裁剪={ok} (重映射={stats['cropped']}) "
f"跳过={skip} "
f"速率={rate:.1f}张/s ETA={eta:.0f}s")
t1 = time.perf_counter()
# ---- 统计 ----
total_ok = stats["cropped"] + stats["nocorners"] + stats["low_overlap"]
total_skip = stats["miss"] + stats["multi"]
print(f"\n{'='*70}")
print(f"完成! {t1-t0:.1f}s")
print(f" 源图: {stats['total']}")
print(f" 裁剪成功: {total_ok} ({total_ok/max(stats['total'],1)*100:.1f}%)")
print(f" - 角点已重映射写入文件名: {stats['cropped']}")
print(f" - 无角点段 (保留原名): {stats['nocorners']}")
print(f" - 重叠不足 (保留原名): {stats['low_overlap']}")
print(f" 跳过: {total_skip}")
print(f" - 漏检: {stats['miss']}")
print(f" - 多框: {stats['multi']}")
print(f"\n 按类别:")
for key in sorted(sc):
s = sc[key]
print(f" {key:30s} ok={s['ok']:5d} skip={s['skip']:5d}")
print(f"\n 输出: {DST_DIR}")
for split in SPLITS:
sp = DST_DIR / split
if sp.exists():
jpg_n = sum(1 for _ in sp.rglob("*.jpg"))
cls_n = len([d for d in sp.iterdir() if d.is_dir()])
print(f" {split}/ : {jpg_n} 图, {cls_n} 类")
print(f"\n{'='*70}")
if __name__ == "__main__":
main()