Installation

git clone https://github.com/happyharrycn/actionformer_release.git
cd actionformer_release
conda create -n actionformer python=3.8
conda activate actionformer
conda install pytorch torchvision -c pytorch
conda install pyyaml h5py joblib tensorboard pandas
cd ./libs/utils
python setup.py install --user
cd ../..
mkdir data

Download thumos14

  1. Put downloaded thumos.tar.gz under the path actionformer_release/data/ and extract it.
  2. Delete the thumos.tar.gz

Train

python ./train.py ./configs/thumos_i3d.yaml --output reproduce

Monitor the training with tensorboard

tensorboard --logdir=./ckpt/thumos_i3d_reproduce/logs

Eval with Debugging

The original author use the below command to evaluate the trained ckpt:

python ./eval.py ./configs/thumos_i3d.yaml ./ckpt/thumos_i3d_reproduce

To check what happens inside the code, create a Run/Debug Configuration for the script eval.py and set the predefined parameters: image

Now start the debug to check what exactly happens inside the codes.

Record of the investigation

Model architecture

FPNIdentity(
  (fpn_norms): ModuleList(
    (0): LayerNorm()
    (1): LayerNorm()
    (2): LayerNorm()
    (3): LayerNorm()
    (4): LayerNorm()
    (5): LayerNorm()
  )
)


PointGenerator(
  (buffer_points): BufferList()
)

PtTransformerClsHead(
  (act): ReLU()
  (head): ModuleList(
    (0): MaskedConv1D(
      (conv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
    )
    (1): MaskedConv1D(
      (conv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
    )
  )
  (norm): ModuleList(
    (0): LayerNorm()
    (1): LayerNorm()
  )
  (cls_head): MaskedConv1D(
    (conv): Conv1d(512, 20, kernel_size=(3,), stride=(1,), padding=(1,))
  )
)

PtTransformerRegHead(
  (act): ReLU()
  (head): ModuleList(
    (0): MaskedConv1D(
      (conv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
    )
    (1): MaskedConv1D(
      (conv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)
    )
  )
  (norm): ModuleList(
    (0): LayerNorm()
    (1): LayerNorm()
  )
  (scale): ModuleList(
    (0): Scale()
    (1): Scale()
    (2): Scale()
    (3): Scale()
    (4): Scale()
    (5): Scale()
  )
  (offset_head): MaskedConv1D(
    (conv): Conv1d(512, 2, kernel_size=(3,), stride=(1,), padding=(1,))
  )
)