bdd.yaml

model:
  arch: video_llama
  model_type: pretrain_llama_v2
  freeze_vit: True
  freeze_qformer: True
  low_resource: False

  # Q-Former
  num_query_token: 32

  # If you want train models based on LLaMA-2-chat,
  # some ckpts could be download from our provided huggingface repo
  # i.e.  https://huggingface.co/DAMO-NLP-SG/Video-LLaMA-2-13B-Finetuned
  llama_model: /input/ckpt/llama-2-7b-chat-hf # "ckpt/llama-2-13b-chat-hf" or "ckpt/vicuna-13b/" or "ckpt/vicuna-7b/" or
  #  imagebind_ckpt_path: # "ckpt/imagebind_huge.pth"

  # The ckpt of vision branch after stage1 pretrained,
  ckpt: /input/ckpt/VL_LLaMA_2_7B_Finetuned.pth # you can use our pretrained ckpt from https://huggingface.co/DAMO-NLP-SG/Video-LLaMA-2-13B-Pretrained/

  # only train vision branch
  equip_audio_branch: False # whether equips the audio branch
  frozen_llama_proj: False
  frozen_video_Qformer: False
  frozen_audio_Qformer: True

  fusion_head_layers: 2
  max_frame_pos: 32
  fusion_header_type: "seqTransf"

  max_txt_len: 320

  # vicuna and llama_2_chat use different template !!!

  # for llama_2_chat:
  end_sym: "</s>"
  prompt_path: "prompts/alignment_image.txt"
  prompt_template: '[INST] <<SYS>>\n \n<</SYS>>\n\n{} [/INST] '

  # for vicuna:
  # end_sym: "###"
  # prompt_path: "prompts/alignment_image.txt"
  # prompt_template: '###Human: {} ###Assistant: '

datasets:
  bdd_instruct:
    data_type: video
    build_info:
      train:
        anno_dir: ./data/BDD_train_data/BDD-Instruct.json # Reaoning/Description Only : BDD-Instruct-reasoning.json/BDD-Instruct-desc.json
        videos_dir: /input/BDD-X-Videos/
    vis_processor:
      train:
        name: "alpro_video_train"
        n_frms: 32
        image_size: 224
    text_processor:
      train:
        name: "blip_caption"
    num_video_query_token: 32
    tokenizer_name: /input/ckpt/llama-2-7b-chat-hf # "ckpt/llama-2-13b-chat-hf" or "ckpt/vicuna-13b/" or "ckpt/vicuna-7b/" or
    model_type: "llama_v2"

  # had_instruct:
  #   data_type: video
  #   build_info:
  #     train:
  #       anno_dir: ./data/HAD_train_data/HAD-instruct-v1.json
  #       videos_dir: ./data/HAD_train_data/videos/ # /input/BDD-X-Videos
  #   vis_processor:
  #     train:
  #       name: "alpro_video_train"
  #       n_frms: 32
  #       image_size: 224
  #   text_processor:
  #     train:
  #       name: "blip_caption"
  #   num_video_query_token: 32
  #   tokenizer_name: /input/ckpt/llama-2-7b-chat-hf # "ckpt/llama-2-13b-chat-hf" or "ckpt/vicuna-13b/" or "ckpt/vicuna-7b/" or
  #   model_type: "llama_v2" # or "vicuna"  # need to set, as vicuna and llama_2_chat use different template

  # drama_instruct:
  #   data_type: video
  #   build_info:
  #     train:
  #       anno_dir: ./data/DRAMA_train_data/DRAMA-Instruct.json
  #       videos_dir: ./data/DRAMA_train_data/video/
  #   vis_processor:
  #     train:
  #       name: "alpro_video_train"
  #       n_frms: 8
  #       image_size: 224
  #   text_processor:
  #     train:
  #       name: "blip_caption"
  #   num_video_query_token: 32
  #   tokenizer_name: /input/ckpt/llama-2-7b-chat-hf # "ckpt/llama-2-13b-chat-hf" or "ckpt/vicuna-13b/" or "ckpt/vicuna-7b/" or
  #   model_type: "llama_v2"

  # maplm_instruct:
  #   data_type: images
  #   build_info:
  #     train:
  #       anno_dir: ./data/MAPLM_train_data/MAPLM-instruct.json
  #       videos_dir: ./data/MAPLM_train_data/video/
  #   vis_processor:
  #     train:
  #       name: "blip2_image_train"
  #       image_size: 224
  #   text_processor:
  #     train:
  #       name: "blip_caption"
  #   num_video_query_token: 8
  #   tokenizer_name: /input/ckpt/llama-2-7b-chat-hf # "ckpt/llama-2-13b-chat-hf" or "ckpt/vicuna-13b/" or "ckpt/vicuna-7b/" or
  #   model_type: "llama_v2"

  # webvid_instruct:
  #   data_type: video
  #   build_info:
  #     anno_dir: ./data/videochat/videochat_instruct_11k.json
  #     videos_dir: path/webvid_align/videos/
  #   vis_processor:
  #     train:
  #       name: "alpro_video_train"
  #       n_frms: 8
  #       image_size: 224
  #   text_processor:
  #     train:
  #       name: "blip_caption"
  #   num_video_query_token: 32
  #   tokenizer_name: "ckpt/llama-2-7b-chat-hf"  # "ckpt/llama-2-13b-chat-hf" or "ckpt/vicuna-13b/" or "ckpt/vicuna-7b/" or
  #   model_type: "llama_v2" # or "vicuna"  # need to set, as vicuna and llama_2_chat use different template

  # cc_sbu_align:
  #   data_type: images
  #     build_info:
  #       storage: path/cc_sbu_align/
  #     vis_processor:
  #       train:
  #         name: "blip2_image_train"
  #         image_size: 224
  #     text_processor:
  #       train:
  #         name: "blip_caption"

  # llava_instruct:
  #   data_type: images
  #   build_info:
  #     anno_dir: path/llava_instruct_150k.json
  #     videos_dir: path/train2014/
  #   vis_processor:
  #     train:
  #       name: "blip2_image_train"
  #       image_size: 224
  #   text_processor:
  #     train:
  #       name: "blip_caption"
  #   num_video_query_token: 32
  #   tokenizer_name: "ckpt/vicuna-13b/" or "ckpt/vicuna-7b/" or "ckpt/llama-2-7b-chat-hf"  or "ckpt/llama-2-13b-chat-hf"
  #   model_type: "llama_v2" or "vicuna"  # need to set, as vicuna and llama_2_chat use different template

run:
  task: video_text_pretrain
  # optimizer
  lr_sched: "linear_warmup_cosine_lr"
  init_lr: 3e-5
  min_lr: 1e-5
  warmup_lr: 1e-6

  weight_decay: 0.05
  max_epoch: 5
  iters_per_epoch: 1000
  batch_size_train: 2
  batch_size_eval: 2
  num_workers: 4
  warmup_steps: 1000

  seed: 42
  output_dir: "./output/"

  amp: True
  resume_ckpt_path: null

  evaluate: False
  train_splits: ["train"]
  # valid_splits: ["val"]

  device: "cuda"
  world_size: 1
  dist_url: "env://"
  distributed: False