Modalを使ってStable Diffusion web UI +ControlNetで遊ぶ

2023年5月23日 01:17

2023/11/12
なんやかんや時間が経って動かないor起動までに時間がかかりすぎるため修正しました。

前提としては修正前同様に偉大なる先駆者のしーぴー様の記事参考に下準備までしておいてください。

ちなみに上記記事の「modalなんてものはない！みたいなエラー」のところについてですが、追加するべきPATHが不明な場合、pip uninstall modal-clientで一回modal-clientをアンインストールして再度インストールすると、warningメッセージとしてどこそこのパス追加しないと動かねぇかもよって警告が出てると思うので、そこに書かれてるパスを追加してください。わかんなかったらwarningメッセージをChat GPTに聞けばパスの追加に関しては結構正確に教えてくれます。
ただし、Modalは比較的最近できたサービスっぽいのでModal自体の操作についてChatGPT聞いても正確に返ってこないと思ったほうがいいでしょう。

そうしましたら下記をstable-diffusion-webui_controlNet3.pyの名前で保存してください。※しーぴー様のを改変させていただきました。

from colorama import Fore
from pathlib import Path

import modal
import shutil
import subprocess
import sys
import shlex
import os

# modal系の変数の定義 ※既存に被っちゃうとあれなので-3にしておきます。
stub = modal.Stub("stable-diffusion-webui-3")
volume_main = modal.NetworkFileSystem.persisted("stable-diffusion-webui-main-3")

# 色んなパスの定義
webui_dir = "/content/stable-diffusion-webui"
webui_model_dir = webui_dir + "/models/Stable-diffusion/"

webui_controlNet_dir = webui_dir + "/extensions/sd-webui-controlnet"
webui_controlNet_model_dir = webui_dir + "/extensions/sd-webui-controlnet/models"

# モデルのID　※model_pathがmodel名と違う場合にも再ダウンロードされないようにmodel_nameを追加しました。あとControlNetがsd15用のためBloodNightOrangeMixを追加
model_ids = [
    {
        "repo_id": "hakurei/waifu-diffusion-v1-4",
        "model_path": "wd-1-4-anime_e1.ckpt",
        "config_file_path": "wd-1-4-anime_e1.yaml",
        "model_name": "wd-1-4-anime_e1.ckpt",
    },
   {
       "repo_id": "WarriorMama777/OrangeMixs",
       "model_path": "Models/BloodOrangeMix/BloodNightOrangeMix.ckpt",
       "config_file_path": "",
       "model_name": "BloodNightOrangeMix.ckpt",
    },
]

# ※v2.6ブランチを使用するのでpythonのバージョン上げたのとrequirements_versions.txtになんとなく合わせました
@stub.function(
    image=modal.Image.from_registry("python:3.10.6-slim")
    .apt_install(
        "git", "libgl1-mesa-dev", "libglib2.0-0", "libsm6", "libxrender1", "libxext6", "gcc", "libcairo2-dev", "aria2",
    )
    .run_commands(
        "pip install -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers"
    )
    .pip_install(
        "blendmodes==2022",
        "transformers==4.30.2",
        "accelerate==0.21.0",
        "basicsr==1.4.2",
        "gfpgan==1.3.8",
        "gradio==3.41.2",
        "numpy==1.23.5",
        "Pillow==9.5.0",
        "realesrgan==0.3.0",
        "torch",
        "omegaconf==2.2.3",
        "pytorch_lightning==1.9.4",
        "scikit-image==0.21.0",
        "fonts",
        "font-roboto",
        "timm==0.9.2",
        "piexif==1.1.3",
        "einops==0.4.1",
        "jsonmerge==1.8.0",
        "clean-fid==0.1.35",
        "resize-right==0.0.2",
        "torchdiffeq==0.2.3",
        "kornia==0.6.7",
        "lark==1.1.2",
        "inflection==0.5.1",
        "GitPython==3.1.32",
        "torchsde==0.2.5",
        "safetensors==0.3.1",
        "httpcore==0.15",
        "tensorboard==2.9.1",
        "taming-transformers==0.0.1",
        "clip",
        "xformers",
        "test-tube",
        "diffusers",
        "invisible-watermark",
        "pyngrok",
        "xformers==0.0.16rc425",
        "gdown",
        "huggingface_hub",
        "colorama",
        "torchmetrics==0.11.4",
        "fastapi==0.94.0",
        "open-clip-torch==2.20.0",
        "psutil==5.9.5",
        "tomesd==0.1.3",
        "httpx==0.24.1",
        
    )
    .pip_install("git+https://github.com/mlfoundations/open_clip.git@bb6e834e9c70d9c27d0dc3ecedeebeaeb1ffad6b"),
    secret=modal.Secret.from_name("my-huggingface-secret"),
    network_file_systems={webui_dir: volume_main},
    gpu="a10g",
    timeout=6000,
)
async def run_stable_diffusion_webui():
    print(Fore.CYAN + "\n---------- セットアップ開始 ----------\n")

    webui_dir_path = Path(webui_model_dir)
    if not webui_dir_path.exists():
        subprocess.run(f"git clone -b v2.6 https://github.com/camenduru/stable-diffusion-webui {webui_dir}", shell=True)

    controlNet_dir_path = Path(webui_controlNet_dir)
    if not controlNet_dir_path.exists():
        subprocess.run(f"git clone https://github.com/Mikubill/sd-webui-controlnet {webui_controlNet_dir}", shell=True)

    # Hugging faceからファイルをダウンロードしてくる関数
    def download_hf_file(repo_id, filename):
        from huggingface_hub import hf_hub_download

        download_dir = hf_hub_download(repo_id=repo_id, filename=filename)
        return download_dir

    # 一部model_nameに修正
    for model_id in model_ids:
        print(Fore.GREEN + model_id["repo_id"] + "のセットアップを開始します...")

        if not Path(webui_model_dir + model_id["model_name"]).exists():
            # モデルのダウンロード＆コピー
            model_downloaded_dir = download_hf_file(
                model_id["repo_id"],
                model_id["model_path"],
            )
            shutil.copy(model_downloaded_dir, webui_model_dir + os.path.basename(model_id["model_path"]))


        if "config_file_path" not in model_id:
          continue

        if not Path(webui_model_dir + model_id["config_file_path"]).exists():
            # コンフィグのダウンロード＆コピー
            config_downloaded_dir = download_hf_file(
                model_id["repo_id"], model_id["config_file_path"]
            )
            shutil.copy(config_downloaded_dir, webui_model_dir + os.path.basename(model_id["config_file_path"]))


        print(Fore.GREEN + model_id["repo_id"] + "のセットアップが完了しました！")

        # controlNetのモデルをダウンロード　※後から導入でも可(後述)。ダウンロードに時間がかかるので必要な場合のみコメント解除してください。
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11e_sd15_ip2p_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11e_sd15_ip2p_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11e_sd15_shuffle_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11e_sd15_shuffle_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11p_sd15_canny_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_canny_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11f1p_sd15_depth_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11f1p_sd15_depth_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11p_sd15_inpaint_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_inpaint_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11p_sd15_lineart_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_lineart_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11p_sd15_mlsd_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_mlsd_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11p_sd15_normalbae_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_normalbae_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11p_sd15_openpose_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_openpose_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11p_sd15_scribble_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_scribble_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11p_sd15_seg_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_seg_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11p_sd15_softedge_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_softedge_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11p_sd15s2_lineart_anime_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15s2_lineart_anime_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/control_v11f1e_sd15_tile_fp16.safetensors", "-d", webui_controlNet_model_dir ,"-o", "control_v11f1e_sd15_tile_fp16.safetensors"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11e_sd15_ip2p_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11e_sd15_ip2p_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11e_sd15_shuffle_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11e_sd15_shuffle_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11p_sd15_canny_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_canny_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11f1p_sd15_depth_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11f1p_sd15_depth_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11p_sd15_inpaint_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_inpaint_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11p_sd15_lineart_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_lineart_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11p_sd15_mlsd_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_mlsd_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11p_sd15_normalbae_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_normalbae_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11p_sd15_openpose_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_openpose_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11p_sd15_scribble_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_scribble_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11p_sd15_seg_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_seg_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11p_sd15_softedge_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15_softedge_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11p_sd15s2_lineart_anime_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11p_sd15s2_lineart_anime_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/raw/main/control_v11f1e_sd15_tile_fp16.yaml", "-d", webui_controlNet_model_dir ,"-o", "control_v11f1e_sd15_tile_fp16.yaml"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/t2iadapter_style_sd14v1.pth", "-d", webui_controlNet_model_dir ,"-o", "t2iadapter_style_sd14v1.pth"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/t2iadapter_sketch_sd14v1.pth", "-d", webui_controlNet_model_dir ,"-o", "t2iadapter_sketch_sd14v1.pth"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/t2iadapter_seg_sd14v1.pth", "-d", webui_controlNet_model_dir ,"-o", "t2iadapter_seg_sd14v1.pth"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/t2iadapter_openpose_sd14v1.pth", "-d", webui_controlNet_model_dir ,"-o", "t2iadapter_openpose_sd14v1.pth"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/t2iadapter_keypose_sd14v1.pth", "-d", webui_controlNet_model_dir ,"-o", "t2iadapter_keypose_sd14v1.pth"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/t2iadapter_depth_sd14v1.pth", "-d", webui_controlNet_model_dir ,"-o", "t2iadapter_depth_sd14v1.pth"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/t2iadapter_color_sd14v1.pth", "-d", webui_controlNet_model_dir ,"-o", "t2iadapter_color_sd14v1.pth"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/t2iadapter_canny_sd14v1.pth", "-d", webui_controlNet_model_dir ,"-o", "t2iadapter_canny_sd14v1.pth"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/t2iadapter_canny_sd15v2.pth", "-d", webui_controlNet_model_dir ,"-o", "t2iadapter_canny_sd15v2.pth"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/t2iadapter_depth_sd15v2.pth", "-d", webui_controlNet_model_dir ,"-o", "t2iadapter_depth_sd15v2.pth"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/t2iadapter_sketch_sd15v2.pth", "-d", webui_controlNet_model_dir ,"-o", "t2iadapter_sketch_sd15v2.pth"])
        #subprocess .run(["aria2c", "--console-log-level=error", "-c", "-x", "16", "-s", "16", "-k", "1M", "https://huggingface.co/ckpt/ControlNet-v1-1/resolve/main/t2iadapter_zoedepth_sd15v1.pth", "-d", webui_controlNet_model_dir ,"-o", "t2iadapter_zoedepth_sd15v1.pth"])

    print(Fore.CYAN + "\n---------- セットアップ完了 ----------\n")

    # WebUIを起動
    sys.path.append(webui_dir)
    sys.argv += shlex.split("--skip-install --xformers")
    os.chdir(webui_dir)
    from launch import start, prepare_environment

    prepare_environment()
    # 最初のargumentは無視されるので注意 ※なんかPythonのライブラリを更新を促されますが調子悪くなりそうだったので一旦--skip-version-checkで飛ばしてます
    sys.argv = shlex.split("--a --gradio-debug --share --xformers --enable-insecure-extension-access --skip-version-check")
    start()


@stub.local_entrypoint()
def main():
    run_stable_diffusion_webui.remote()

保存したら下記みたいな感じでターミナルとかコマンドプロンプトとかで

modal run stable-diffusion-webui_controlNet3.py

ってやれば大丈夫なはず。
最初はそこそこ時間がかかりますが気長に待ちましょう。
今回は起動優先でcontrolNetのモデルダウンロードは後回しにしてます。
reference onlyはモデルのダウンロードなしでそのまま使えるので一旦動作確認はreference onlyでやって、大丈夫そうなら他のモデルも足すのがいいかなと思います。
ちなみに画像生成の方のモデルなんですが、BloodNightOrangeMixだけにしたらうまくいかなかったのでwd-1-4-anime_e1も残してます(コンフィグの問題かなぁ←よくわかってない)

参考にしたもの

※colabは無料では今は警告が出ます。

--enable-insecure-extension-accessについて

そうだ、あと↑の方のModalおすすめやでの記事で配布されてるコードのままだと拡張機能か何かが使えなくて、付け足した部分があります。 pic.twitter.com/qaAlCGiP5w
— 令五 (@reiwagonen) April 27, 2023

こちらのツイートで助かりました。

途中困ったこと

Cloning Stable Diffusion into repositories/stable-diffusion-stability-ai...

みたいなクローン関係が遅い、終わらない。
→実行した時間帯によってはめちゃくちゃ時間がかかる。待つしかない。
modal側のネット環境の問題？

ControlNetのモデルのダウンロードが遅い
→一回ローカルにダウンロードしてputコマンドを使った方が早い。
ローカル環境にcn_modelsというフォルダ(フォルダ名は任意で良い)を作成してそこにControlNetの使いたいモデルを配置して下記コードを実行する

modal nfs put stable-diffusion-webui-main-3 自分のローカルパス/cn_models /extensions/sd-webui-controlnet/models

これでローカルのcn_modelsフォルダの中身が、modalのmodelsフォルダへアップロードされます。
※同様に画像生成の方のモデルやLoraのモデルの追加にも使用できますので適宜modal側のリモートパスを書き換えて実行してみてください。
例）stable-diffusionのモデルの場合(sd_modelsにckptとかを格納してアップ)

modal nfs put stable-diffusion-webui-main-3 自分のローカルパス/sd_models /models/Stable-diffusion

で追加できます。

*** Failed reading extension data from Git repository (sd-webui-controlnet)
Traceback (most recent call last):〜

というエラーが出る
→どうやらGit側の問題？再起動したら出なくなった。特に修正は不要っぽい。

こんな感じですかねぇ。。まだまだ不安定なのでまたいつ動かなくなるかわからんですが。
pythonよくわかってない中とりあえず動けばいいや精神でやってるのでおかしなところとかあったらすみません。
暖かい目で見守ってください。

この記事が気に入ったらサポートをしてみませんか？