Editing Layers of Tabular Learner?

fastai
1

How does one edit the layers of a Tabular Learner? Specifically, I am doing multiout regression but I want the activations to sum to 1 (because in my case the targets are fractions that should add up to 1). By default (or at least when you specify a y_range) the Tabular Learner uses a Sigmoid as its final activation, but I would like to change this to a softmax, so that the outputs sum to 1 (model output shown below). How can I change this final layer to a softmax? Alternatively, could I not specify a y_range (and therefore not get a sigmoid at the end) and then append a softmax function as the final layer?

y_labels['fraction1', 'fraction2', 'fraction3', 'fraction4', 'fraction5']
procs = [Normalize]
to = TabularPandas(df,
                   procs=procs,
                   cat_names = [],
                   cont_names = feature_labels,
                   y_names=y_labels,
                   splits=splits,
                   y_block = RegressionBlock(n_out=len(y_labels)))

dls = to.dataloaders(bs=512, num_workers=0)
learn = tabular_learner(dls, layers=[500,250], metrics=[mse, rmse, mae], y_range = (0.0,1.0), n_out=len(y_labels))

learn.model

TabularModel(
  (embeds): ModuleList()
  (emb_drop): Dropout(p=0.0, inplace=False)
  (bn_cont): BatchNorm1d(125, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (layers): Sequential(
    (0): LinBnDrop(
      (0): BatchNorm1d(125, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (1): Linear(in_features=125, out_features=500, bias=False)
      (2): ReLU(inplace=True)
    )
    (1): LinBnDrop(
      (0): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (1): Linear(in_features=500, out_features=250, bias=False)
      (2): ReLU(inplace=True)
    )
    (2): LinBnDrop(
      (0): Linear(in_features=250, out_features=5, bias=True)
    )
    (3): SigmoidRange(low=0.0, high=1.0)
  )
)
2

Just write your own version of the TabularModel in torch would be the easiest way, then mimic what tabular_learner does to build it (see tabular_learner??), and then make your learner with Learner(dls, myNewModel, metrics=...)

1 Like
3

That worked perfectly, thank you!

In case any one else is curious here are my new versions of tabular_learner and TabularModel:

class my_TabularModel(Module):
    "My model for tabular data."
    def __init__(self, emb_szs, n_cont, out_sz, layers, ps=None, embed_p=0.,
                 y_range=None, use_bn=True, bn_final=False, bn_cont=True, act_cls=nn.ReLU(inplace=True), use_softmax=False):
        ps = ifnone(ps, [0]*len(layers))
        if not is_listy(ps): ps = [ps]*len(layers)
        self.embeds = nn.ModuleList([Embedding(ni, nf) for ni,nf in emb_szs])
        self.emb_drop = nn.Dropout(embed_p)
        self.bn_cont = nn.BatchNorm1d(n_cont) if bn_cont else None
        n_emb = sum(e.embedding_dim for e in self.embeds)
        self.n_emb,self.n_cont = n_emb,n_cont
        sizes = [n_emb + n_cont] + layers + [out_sz]
        actns = [act_cls for _ in range(len(sizes)-2)] + [None]
        _layers = [LinBnDrop(sizes[i], sizes[i+1], bn=use_bn and (i!=len(actns)-1 or bn_final), p=p, act=a) for i,(p,a) in enumerate(zip(ps+[0.],actns))]
        if (y_range is not None)&(use_softmax==False): 
            _layers.append(SigmoidRange(*y_range))
        if use_softmax==True: 
            _layers.append(nn.Softmax())
        self.layers = nn.Sequential(*_layers)

    def forward(self, x_cat, x_cont=None):
        if self.n_emb != 0:
            x = [e(x_cat[:,i]) for i,e in enumerate(self.embeds)]
            x = torch.cat(x, 1)
            x = self.emb_drop(x)
        if self.n_cont != 0:
            if self.bn_cont is not None: x_cont = self.bn_cont(x_cont)
            x = torch.cat([x, x_cont], 1) if self.n_emb != 0 else x_cont
        return self.layers(x)
    
def my_tabular_learner(dls, layers=None, emb_szs=None, config=None, n_out=None, y_range=None, use_softmax=False, **kwargs):
    "Get a `Learner` using `dls`, with `metrics`, including a `TabularModel` created using the remaining params."
    if config is None: config = tabular_config()
    if layers is None: layers = [200,100]
    to = dls.train_ds
    emb_szs = get_emb_sz(dls.train_ds, {} if emb_szs is None else emb_szs)
    if n_out is None: n_out = get_c(dls)
    assert n_out, "`n_out` is not defined, and could not be inferred from data, set `dls.c` or pass `n_out`"
    if y_range is None and 'y_range' in config: y_range = config.pop('y_range')
    model = my_TabularModel(emb_szs, len(dls.cont_names), n_out, layers, y_range=y_range,use_softmax=use_softmax, **config)
    return TabularLearner(dls, model, **kwargs)
1 Like