[Question]: 多机微调模型时出现直接卡在加载过后，计算loss之前

[1190303125]

请提出你的问题

下面是我的环境：

name: paddle_new
channels:
  - conda-forge
  - defaults
dependencies:
  - _libgcc_mutex=0.1=conda_forge
  - _openmp_mutex=4.5=2_gnu
  - binutils_impl_linux-64=2.40=h5293946_0
  - bzip2=1.0.8=h5eee18b_6
  - ca-certificates=2024.9.24=h06a4308_0
  - cuda-version=12.4=hbda6634_3
  - gcc=14.2.0=h96c4ede_1
  - gcc_impl_linux-64=14.2.0=h6b349bd_1
  - kernel-headers_linux-64=3.10.0=he073ed8_18
  - ld_impl_linux-64=2.40=h12ee557_0
  - libffi=3.4.4=h6a678d5_1
  - libgcc=14.2.0=h77fa898_1
  - libgcc-devel_linux-64=14.2.0=h41c2201_101
  - libgcc-ng=14.2.0=h69a702a_1
  - libgomp=14.2.0=h77fa898_1
  - libsanitizer=14.2.0=h2a3dede_1
  - libstdcxx=14.2.0=hc0a3c3a_1
  - libstdcxx-ng=11.2.0=h1234567_1
  - libuuid=1.41.5=h5eee18b_0
  - nccl=2.23.4.1=h52f6c39_2
  - ncurses=6.4=h6a678d5_0
  - openssl=3.4.0=hb9d3cd8_0
  - pip=24.2=py310h06a4308_0
  - python=3.10.15=he870216_1
  - readline=8.2=h5eee18b_0
  - setuptools=75.1.0=py310h06a4308_0
  - sqlite=3.45.3=h5eee18b_0
  - sysroot_linux-64=2.17=h4a8ded7_18
  - tk=8.6.14=h39e8969_0
  - wheel=0.44.0=py310h06a4308_0
  - xz=5.4.6=h5eee18b_1
  - zlib=1.2.13=h5eee18b_1
  - pip:
      - aiohappyeyeballs==2.4.3
      - aiohttp==3.11.7
      - aiosignal==1.3.1
      - aistudio-sdk==0.2.6
      - annotated-types==0.7.0
      - anyio==3.7.1
      - astor==0.8.1
      - async-timeout==5.0.1
      - attrs==24.2.0
      - babel==2.16.0
      - bce-python-sdk==0.9.23
      - bcrypt==4.2.1
      - blinker==1.9.0
      - brotli==1.1.0
      - certifi==2024.8.30
      - cffi==1.17.1
      - charset-normalizer==3.4.0
      - click==8.1.7
      - colorama==0.4.6
      - colorlog==6.9.0
      - contourpy==1.3.1
      - cryptography==43.0.3
      - cuda-python==12.6.2.post1
      - cycler==0.12.1
      - datasets==3.1.0
      - decorator==5.1.1
      - dill==0.3.4
      - exceptiongroup==1.2.2
      - fast-dataindex==0.1.2
      - fastapi==0.115.5
      - ffmpy==0.4.0
      - filelock==3.16.1
      - flask==3.1.0
      - flask-babel==4.0.0
      - fonttools==4.55.0
      - frozenlist==1.5.0
      - fsspec==2024.9.0
      - future==1.0.0
      - gevent==24.11.1
      - geventhttpclient==2.3.3
      - gradio==3.11.0
      - greenlet==3.1.1
      - grpcio==1.67.1
      - h11==0.12.0
      - httpcore==0.15.0
      - httpx==0.25.1
      - huggingface-hub==0.26.2
      - idna==3.10
      - itsdangerous==2.2.0
      - jieba==0.42.1
      - jinja2==3.1.4
      - joblib==1.4.2
      - kiwisolver==1.4.7
      - linkify-it-py==2.0.3
      - markdown-it-py==3.0.0
      - markupsafe==3.0.2
      - matplotlib==3.10.0rc1
      - mdit-py-plugins==0.4.2
      - mdurl==0.1.2
      - mpmath==1.3.0
      - multidict==6.1.0
      - multiprocess==0.70.12.2
      - networkx==3.4.2
      - numpy==1.26.4
      - nvidia-cublas-cu11==11.11.3.6
      - nvidia-cuda-cupti-cu11==11.8.87
      - nvidia-cuda-nvrtc-cu11==11.8.89
      - nvidia-cuda-runtime-cu11==11.8.89
      - nvidia-cudnn-cu11==8.7.0.84
      - nvidia-cufft-cu11==10.9.0.58
      - nvidia-curand-cu11==10.3.0.86
      - nvidia-cusolver-cu11==11.4.1.48
      - nvidia-cusparse-cu11==11.7.5.86
      - nvidia-nccl-cu11==2.19.3
      - nvidia-nccl-cu12==2.23.4
      - nvidia-nvtx-cu11==11.8.86
      - onnx==1.17.0
      - opt-einsum==3.3.0
      - orjson==3.10.12
      - packaging==24.2
      - paddle2onnx==1.3.0
      - paddlefsl==1.1.0
      - paddlenlp==2.8.0.post0
      - paddlepaddle-gpu==3.0.0b1
      - pandas==2.2.3
      - paramiko==3.5.0
      - pillow==11.0.0
      - prettytable==3.12.0
      - propcache==0.2.0
      - protobuf==5.28.3
      - psutil==6.1.0
      - pyarrow==18.1.0
      - pybind11==2.14.0.dev1
      - pycparser==2.22
      - pycryptodome==3.21.0
      - pydantic==2.10.1
      - pydantic-core==2.27.1
      - pydub==0.25.1
      - pygments==2.18.0
      - pynacl==1.5.0
      - pyparsing==3.2.0
      - python-dateutil==2.9.0.post0
      - python-multipart==0.0.17
      - python-rapidjson==1.20
      - pytz==2024.2
      - pyyaml==6.0.2
      - rarfile==4.2
      - regex==2024.11.6
      - requests==2.32.3
      - rich==13.9.4
      - safetensors==0.4.6.dev0
      - scikit-learn==1.6.0rc1
      - scipy==1.14.1
      - sentencepiece==0.2.0
      - shellingham==1.5.4
      - six==1.16.0
      - sniffio==1.3.1
      - starlette==0.41.3
      - sympy==1.13.3
      - threadpoolctl==3.5.0
      - tiktoken==0.8.0
      - tokenizers==0.20.3
      - tool-helpers==0.1.2
      - tqdm==4.67.1
      - tritonclient==2.52.0
      - typer==0.13.1
      - typing-extensions==4.12.2
      - tzdata==2024.2
      - uc-micro-py==1.0.3
      - urllib3==2.2.3
      - uvicorn==0.32.1
      - visualdl==3.0.0b0
      - wcwidth==0.2.13
      - websockets==14.1
      - werkzeug==3.1.3
      - x2paddle==1.5.0
      - xxhash==3.5.0
      - yarl==1.18.0
      - zope-event==5.0
      - zope-interface==7.1.1

卡死在以下这个位置：

<unk> 0
[2024-11-27 22:25:23,401] [    INFO] - Starting training from resume_from_checkpoint : None
n2:8140:8140 [0] NCCL INFO Bootstrap : Using ib0.8079:192.168.1.3<0>
n2:8140:8140 [0] NCCL INFO cudaDriverVersion 12000
n2:8140:8140 [0] NCCL INFO NCCL version 2.23.4+cuda12.0
n2:8140:8465 [0] NCCL INFO NET/Plugin: Could not find: libnccl-net.so. Using internal network plugin.
n2:8140:8465 [0] NCCL INFO NET/IB : Using [0]mlx5_2:1/IB [1]mlx5_0:1/IB [2]mlx5_3:1/IB [3]mlx5_1:1/IB ; OOB ib0.8079:192.168.1.3<0>
n2:8140:8465 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
n2:8140:8465 [0] NCCL INFO Using network IB
n2:8140:8465 [0] NCCL INFO ncclCommInitRank comm 0xaf438e60 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 10000 commId 0x74182825db762150 - Init START

[wawltor]

可以用nccl-test来检查下多机互联的情况 https://github.com/NVIDIA/nccl-tests