load bert pretrained weight to detr encoder

# Example python script of loading BERT-base model to DETR



# Create DETR

import argparse

from models import build_model

from util.default_args import get_args_parser

parser = argparse.ArgumentParser(parents=[get_args_parser()])

args = parser.parse_known_args()[0]

args.model = "detr"

args.hidden_dim = 768

args.dim_feedforward = 3072

args.lr = 1e-4

args.lr_backbone = 1e-5

args.num_queries = 100

args.enc_layers = 12

args.nheads = 12

model, criterion, postprocessors = build_model(args)


# Load BERT

import torch

bert = torch.hub.load('huggingface/pytorch-transformers', 'model', 'bert-base-uncased')

bert_enc = bert.encoder.state_dict()


# Convert keys

dict_bert2detr = {}

for i in range(args.enc_layers):

    key = "layers.{}.self_attn.in_proj_weight".format(i)

    dict_bert2detr[key] = torch.cat([bert_enc["layer.{}.attention.self.query.weight".format(i)], [bert_enc["layer.{}.attention.self.key.weight".format(i)], [bert_enc["layer.{}.attention.self.value.weight".format(i)]], dim=0)

    key = "layers.{}.self_attn.in_proj_bias".format(i)

    dict_bert2detr[key] = torch.cat([bert_enc["layer.{}.attention.self.query.bias".format(i)], [bert_enc["layer.{}.attention.self.key.bias".format(i)], [bert_enc["layer.{}.attention.self.value.bias".format(i)]], dim=0)

    key = "layers.{}.self_attn.out_proj.weight".format(i)

    dict_bert2detr[key] = bert_enc["layer.{}.attention.output.dense.weight".format(i)]

    key = "layers.{}.self_attn.out_proj.bias".format(i)

    dict_bert2detr[key] = bert_enc["layer.{}.attention.output.dense.bias".format(i)]

    key = "layers.{}.linear1.weight".format(i)

    dict_bert2detr[key] = bert_enc["layer.{}.intermediate.dense.weight".format(i)]

    key = "layers.{}.linear1.bias".format(i)

    dict_bert2detr[key] = bert_enc["layer.{}.intermediate.dense.bias".format(i)]

    key = "layers.{}.linear2.weight".format(i)

    dict_bert2detr[key] = bert_enc["layer.{}.output.dense.weight".format(i)]

    key = "layers.{}.linear2.bias".format(i)

    dict_bert2detr[key] = bert_enc["layer.{}.output.dense.bias".format(i)]

    key = "layers.{}.norm1.weight".format(i)

    dict_bert2detr[key] = bert_enc["layer.{}.attention.output.LayerNorm.weight".format(i)]

    key = "layers.{}.norm1.bias".format(i)

    dict_bert2detr[key] = bert_enc["layer.{}.attention.output.LayerNorm.bias".format(i)]

    key = "layers.{}.norm2.weight".format(i)

    dict_bert2detr[key] = bert_enc["layer.{}.output.LayerNorm.weight".format(i)]

    key = "layers.{}.norm2.bias".format(i)

    dict_bert2detr[key] = bert_enc["layer.{}.output.LayerNorm.bias".format(i)]


# Load BERT weights into DETR

model.transformer.encoder.load_state_dict(dict_bert2detr)

댓글

댓글 쓰기

이 블로그의 인기 게시물

sklearn tsne + matplotlib scatter

import os import torch import clip import matplotlib . pyplot as plt from PIL import Image from sklearn . manifold import TSNE from tqdm import tqdm from collections import defaultdict from random import shuffle   X = torch . cat ( X , dim = 0 ) X_emb = TSNE ( init = "pca" , perplexity = 30.0 ). fit_transform ( X . cpu ()) labels = [ "source" , "z_star/foggy-1" , "z_star/foggy-2" , "z_star/foggy-3" , "z_bar/foggy-1" , "z_bar/foggy-2" , "z_bar/foggy-3" ] fig , ax = plt . subplots ( 1 ) group_len = len ( zs ) for i in range ( 7 ): ax . scatter ( X_emb [ i * group_len :( i + 1 )* group_len , 0 ], X_emb [ i * group_len :( i + 1 )* group_len , 1 ], label = labels [ i ], s = 4 ) ax . legend () fig .savefig( "temp.png" )
자세한 내용 보기

Implementation of Focal Loss using Pytorch

import torch import torch.nn as nn class BinaryFocalLossWithLogits(nn.Module): def __init__ ( self , gamma= 0 , alpha_dominant= None, reduce= True ): super (BinaryFocalLossWithLogits , self ). __init__ () self .gamma = gamma self .alpha = alpha_dominant self .reduce = reduce self .BCELoss = nn.BCEWithLogitsLoss( reduction = 'mean' if self .reduce else 'none' ) def forward ( self , inputs , labels): loss = self .BCELoss(inputs , labels) pt = torch.exp(- 1 * loss) at = torch.tensor( list ( map ( lambda x: self .alpha if x == 1 else 1 - self .alpha , labels.view(- 1 ).tolist()))) loss = loss * (( 1 - pt) ** self .gamma) * at if self .reduce: return torch.mean(loss) else : return loss usage: criterion = BinaryFocalLossWithLogits( gamma = 2 , alpha_dominant = 0.1 , reduce = False ) labels = torch.tensor([[ 1. , 0. , 0. ]]) output = torch.tensor([[- 0...
자세한 내용 보기