完善部署并训练5个新发型 + 换发型集成文档

部署修复:
- torch.load 增加 weights_only=False patch,兼容 PyTorch 2.6+ 加载旧权重
- OSS 改为懒加载,本地用 output_format=base64 无需配凭证即可启动
- 补全被 gitignore 误排除的必需代码:core/models/layers/data、models/layers/data、keypoints/lib
- webui 训练命令 --xformers 改 --sdpa(修复 xformers 无 CUDA 支持报错)

功能调整:
- hair_grow_service 端口改 8899、preview 路由修复(send_file)
- list_hairstyles 增加发型白名单,测试页只展示当前5个发型

新增脚本:
- train_lora_parallel.py:直接调 kohya 并行训练 LoRA(绕过 photo_service 串行限制)
- train_hairstyles_parallel.py / train_batch_stepC.py:批量训练辅助脚本
- scripts/sync_data_to_server.sh:大文件断点续传到云服务器

文档:
- docs/换发型集成文档.md:换发型完整流程、服务架构、资源依赖、训练方法、集成步骤
This commit is contained in:
xsl
2026-07-09 20:45:08 +08:00
parent aaa1bf159f
commit ce445b64a6
42 changed files with 10553 additions and 34 deletions
+124
View File
@@ -0,0 +1,124 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""直接调 kohya train_network.py 真正并行训练 LoRA
绕过 photo_service 的串行阻塞,用 subprocess 直接起 train_network.py 进程,
通过 ThreadPoolExecutor 控制并发数。
用法:
python /home/xsl/change_hair/train_lora_parallel.py
前提:step1 训练数据已准备好(data/train_material/<hid>/images/1_hairstyle/*.png
"""
import os
import sys
import time
import subprocess
from concurrent.futures import ThreadPoolExecutor, as_completed
TRAIN_DIR = "/home/xsl/change_hair/data/train_material"
KOHYA_WORKDIR = "/home/xsl/change_hair/project/kohya_ss_home/kohya_ss"
BASE_MODEL = "/home/xsl/change_hair/project/onediff/stable-diffusion-webui/models/Stable-diffusion/v1-5-pruned-emaonly.safetensors"
KOHYA_ACCEL = "/home/xsl/miniconda3/envs/kohya/bin/accelerate"
LOG_DIR = "/home/xsl/change_hair/project/logs"
# 本次要训练的发型
HAIRSTYLES = ["chang_tuoyuan", "chang_zhixian", "chang_huaban", "chang_xinxing"]
# chang_bolang 已完成,跳过
PARALLEL = 2
def log(msg):
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
def train_one(hair_id):
"""训练单个发型的 LoRA,返回 (hair_id, success, log_path)"""
images_dir = os.path.join(TRAIN_DIR, hair_id, "images")
model_dir = os.path.join(TRAIN_DIR, hair_id, "model")
os.makedirs(model_dir, exist_ok=True)
# 准备 sample prompt 文件
sample_dir = os.path.join(model_dir, "sample")
os.makedirs(sample_dir, exist_ok=True)
sample_txt = os.path.join(sample_dir, "prompt.txt")
with open(sample_txt, "w") as f:
f.write('titor hairstyle, easyphoto, faceless, no human, white background, simple background, '
' --n low quality, worst quality, bad anatomy, bad composition, poor, low effort --h 768 '
'--w 768 --s 30 --l 7')
log_path = os.path.join(LOG_DIR, f"train_{hair_id}.log")
cmd = [
"bash", "-c",
f'cd {KOHYA_WORKDIR} && '
f'HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 CUDA_VISIBLE_DEVICES=0 '
f'{KOHYA_ACCEL} launch --num_cpu_threads_per_process=2 "./train_network.py" --enable_bucket '
f'--min_bucket_reso=256 --max_bucket_reso=2048 --pretrained_model_name_or_path="{BASE_MODEL}" '
f'--train_data_dir={images_dir} --resolution="2000,2000" '
f'--output_dir={model_dir} '
'--network_alpha="64" --save_model_as=safetensors --network_module=networks.lora --text_encoder_lr=5e-05 '
'--unet_lr=0.0001 --network_dim=128 --output_name="hairstyle_hd_lora" --lr_scheduler_num_cycles="20" '
'--no_half_vae --learning_rate="0.0001" --lr_scheduler="cosine" --lr_warmup_steps="650" --train_batch_size="1" '
'--max_train_steps="1500" --save_every_n_epochs="100" --mixed_precision="fp16" --save_precision="fp16" '
'--caption_extension=".txt" --sample_sampler=ddim '
f'--sample_prompts={sample_txt} --sample_every_n_epochs="1000" '
'--seed="1234" --cache_latents --optimizer_type="AdamW" --max_data_loader_n_workers="0" --bucket_reso_steps=64 '
'--sdpa --bucket_no_upscale --noise_offset=0.0'
]
log(f"{hair_id} 开始训练,日志: {log_path}")
t0 = time.time()
try:
with open(log_path, "w") as lf:
proc = subprocess.run(cmd, stdout=lf, stderr=subprocess.STDOUT, timeout=1800)
ok = proc.returncode == 0
lora = os.path.join(model_dir, "hairstyle_hd_lora.safetensors")
ok = ok and os.path.exists(lora) and os.path.getsize(lora) > 100000000
elapsed = time.time() - t0
sz = os.path.getsize(lora) // 1024 // 1024 if os.path.exists(lora) else 0
log(f" {'' if ok else ''} {hair_id} {'训练完成' if ok else '训练失败'} ({sz}MB, {elapsed:.0f}s)")
return (hair_id, ok, log_path)
except subprocess.TimeoutExpired:
log(f"{hair_id} 训练超时")
return (hair_id, False, log_path)
except Exception as e:
log(f"{hair_id} 训练异常: {e}")
return (hair_id, False, log_path)
def main():
t0 = time.time()
log(f"并行 LoRA 训练: {HAIRSTYLES} (并发={PARALLEL})")
results = {}
with ThreadPoolExecutor(max_workers=PARALLEL) as ex:
futures = {ex.submit(train_one, hid): hid for hid in HAIRSTYLES}
for fut in as_completed(futures):
hid = futures[fut]
try:
hair_id, ok, log_path = fut.result()
results[hair_id] = ok
except Exception as e:
log(f"{hid} 异常: {e}")
results[hid] = False
log(f"\n{'='*60}")
log(f"全部完成,耗时 {time.time()-t0:.0f}s")
for hid, ok in results.items():
lora = os.path.join(TRAIN_DIR, hid, "model", "hairstyle_hd_lora.safetensors")
sz = os.path.getsize(lora) // 1024 // 1024 if os.path.exists(lora) else 0
log(f" {'' if ok else ''} {hid} ({sz}MB)")
success = [h for h, ok in results.items() if ok]
log(f"成功: {success}")
# 写完成清单
done_file = os.path.join(LOG_DIR, "train_lora_done.json")
import json
with open(done_file, "w") as f:
json.dump({"completed": success, "all": HAIRSTYLES}, f)
log(f"完成清单: {done_file}")
if __name__ == "__main__":
main()