print("""
neuron dropout: p=0.01, rank dropout: p=None, module dropout: p=None
create LoRA for Text Encoder 1:
create LoRA for Text Encoder 2:
create LoRA for Text Encoder: 264 modules.
create LoRA for U-Net: 722 modules.
enable LoRA for text encoder
enable LoRA for U-Net
prepare optimizer, data loader etc.
use 8-bit AdamW optimizer | {}
running training
  num train images * repeats: 509
  num reg images: 0
  num batches per epoch: 509
  num epochs: 10
  batch size per device: 1
  gradient accumulation steps = 1
  total optimization steps: 5090
steps:   0%|                                        | 0/5090 [00:00<?, ?it/s] epoch 1/10""") print('File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl') print(" return forward_call(*args, **kwargs)") print('File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/activation.py", line 396, in forward') print(' return F.silu(input, inplace=self.inplace)') print('File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/functional.py", line 2059, in silu') print(' return torch._C_nn.silu(input)') print('\033[93mtorch.cuda.OutOfMoneyError:\033[0m \033[91mCUDA out of money.\033[0m Tried to allocate \033[93m$156.00\033[0m (GPU 0; \033[92m$47360.00\033[0m total capacity; \033[91m$45740.00\033[0m already allocated; \033[92m$0.00\033[0m free; \033[96m$46890.00\033[0m reserver in total by \033[95mJensen Huang\033[0m) If reserverd money is >> allocated moeny try setting max_split_size_cent to avoid fragmentation.\nSee \033[92mCOMPUTEX TAIPEI 2024\033[0m for Money Management and NV_SHOP_LIST')
print('steps:    0%|                                        | 1/5090 [00:06<3:40:17, 6.75s/it, avr_loss=0.17]')
print('\033[96m11:59:56-667672\033[0m \033[91mERROR\033[0m    Training failed')
input()

Categories:

Tags:

No responses yet

发表回复

您的邮箱地址不会被公开。 必填项已用 * 标注

近期评论
归档
分类