200814,test bert 001

In [1]:

# conda install -c conda-forge ipywidgets

In [2]:

from fast_bert.data_cls import BertDataBunch


DATA_PATH  = './../Downloads/fastai/fast-bert-1.8.0/sample_data/imdb_movie_reviews/data/'
LABEL_PATH = './../Downloads/fastai/fast-bert-1.8.0/sample_data/imdb_movie_reviews/label/'
OUTPUT_DIR = './../Downloads/fastai/fast-bert-1.8.0/sample_data/imdb_movie_reviews/output/'

databunch = BertDataBunch(DATA_PATH, LABEL_PATH,
                          tokenizer='bert-base-uncased',
                          train_file='train_sample.csv',
                          val_file='val_sample.csv',
                          label_file='labels.csv',
                          text_col='text',
                          label_col='label',
                          batch_size_per_gpu=16,
                          max_seq_length=512,
                          multi_gpu=True,
                          multi_label=False,
                          model_type='bert')

D:\app_tool\anaconda\envs\tf114\lib\site-packages\tensorflow\python\framework\dtypes.py:523: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint8 = np.dtype([("qint8", np.int8, 1)])
D:\app_tool\anaconda\envs\tf114\lib\site-packages\tensorflow\python\framework\dtypes.py:524: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
D:\app_tool\anaconda\envs\tf114\lib\site-packages\tensorflow\python\framework\dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint16 = np.dtype([("qint16", np.int16, 1)])
D:\app_tool\anaconda\envs\tf114\lib\site-packages\tensorflow\python\framework\dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_quint16 = np.dtype([("quint16", np.uint16, 1)])
D:\app_tool\anaconda\envs\tf114\lib\site-packages\tensorflow\python\framework\dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint32 = np.dtype([("qint32", np.int32, 1)])
D:\app_tool\anaconda\envs\tf114\lib\site-packages\tensorflow\python\framework\dtypes.py:532: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  np_resource = np.dtype([("resource", np.ubyte, 1)])

In [4]:

databunch
# Create a Learner Object
from fast_bert.learner_cls import BertLearner
from fast_bert.metrics import accuracy
import logging
import torch


logger = logging.getLogger()
device_cuda = torch.device("cuda")
metrics = [{'name': 'accuracy', 'function': accuracy}]

learner = BertLearner.from_pretrained_model(
                        databunch,
                        pretrained_path='bert-base-uncased',
                        metrics=metrics,
                        device=device_cuda,
                        logger=logger,
                        output_dir=OUTPUT_DIR,
                        finetuned_wgts_path=None,
                        warmup_steps=500,
                        multi_gpu=False,
                        is_fp16=True,
                        multi_label=False,
                        logging_steps=50)

Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']
- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).
- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
<ipython-input-4-4b0b96630dce> in <module>
     23                         is_fp16=True,
     24                         multi_label=False,
---> 25                         logging_steps=50)

D:\app_tool\anaconda\envs\tf114\lib\site-packages\fast_bert\learner_cls.py in from_pretrained_model(dataBunch, pretrained_path, output_dir, metrics, device, logger, finetuned_wgts_path, multi_gpu, is_fp16, loss_scale, warmup_steps, fp16_opt_level, grad_accumulation_steps, multi_label, max_grad_norm, adam_epsilon, logging_steps, freeze_transformer_layers)
    190 
    191         model = load_model(
--> 192             dataBunch, pretrained_path, finetuned_wgts_path, device, multi_label
    193         )
    194 

D:\app_tool\anaconda\envs\tf114\lib\site-packages\fast_bert\learner_cls.py in load_model(dataBunch, pretrained_path, finetuned_wgts_path, device, multi_label)
    160         )
    161 
--> 162     return model.to(device)
    163 
    164 

D:\app_tool\anaconda\envs\tf114\lib\site-packages\torch\nn\modules\module.py in to(self, *args, **kwargs)
    423             return t.to(device, dtype if t.is_floating_point() else None, non_blocking)
    424 
--> 425         return self._apply(convert)
    426 
    427     def register_backward_hook(self, hook):

D:\app_tool\anaconda\envs\tf114\lib\site-packages\torch\nn\modules\module.py in _apply(self, fn)
    199     def _apply(self, fn):
    200         for module in self.children():
--> 201             module._apply(fn)
    202 
    203         def compute_should_use_set_data(tensor, tensor_applied):

D:\app_tool\anaconda\envs\tf114\lib\site-packages\torch\nn\modules\module.py in _apply(self, fn)
    199     def _apply(self, fn):
    200         for module in self.children():
--> 201             module._apply(fn)
    202 
    203         def compute_should_use_set_data(tensor, tensor_applied):

D:\app_tool\anaconda\envs\tf114\lib\site-packages\torch\nn\modules\module.py in _apply(self, fn)
    199     def _apply(self, fn):
    200         for module in self.children():
--> 201             module._apply(fn)
    202 
    203         def compute_should_use_set_data(tensor, tensor_applied):

D:\app_tool\anaconda\envs\tf114\lib\site-packages\torch\nn\modules\module.py in _apply(self, fn)
    221                 # `with torch.no_grad():`
    222                 with torch.no_grad():
--> 223                     param_applied = fn(param)
    224                 should_use_set_data = compute_should_use_set_data(param, param_applied)
    225                 if should_use_set_data:

D:\app_tool\anaconda\envs\tf114\lib\site-packages\torch\nn\modules\module.py in convert(t)
    421 
    422         def convert(t):
--> 423             return t.to(device, dtype if t.is_floating_point() else None, non_blocking)
    424 
    425         return self._apply(convert)

D:\app_tool\anaconda\envs\tf114\lib\site-packages\torch\cuda\__init__.py in _lazy_init()
    194             raise RuntimeError(
    195                 "Cannot re-initialize CUDA in forked subprocess. " + msg)
--> 196         _check_driver()
    197         torch._C._cuda_init()
    198         _cudart = _load_cudart()

D:\app_tool\anaconda\envs\tf114\lib\site-packages\torch\cuda\__init__.py in _check_driver()
     92 def _check_driver():
     93     if not hasattr(torch._C, '_cuda_isDriverSufficient'):
---> 94         raise AssertionError("Torch not compiled with CUDA enabled")
     95     if not torch._C._cuda_isDriverSufficient():
     96         if torch._C._cuda_getDriverVersion() == 0:

AssertionError: Torch not compiled with CUDA enabled

In [ ]:

# 3. Find the optimal learning rate
learner.lr_find(start_lr=1e-5,optimizer_type='lamb')