Suppose I already have pure PyTorch code defining datset + dataloader. I want to use fastai v2 learner to train the model. See Jeremy’s blog
https://www.fast.ai/2020/02/13/fastai-A-Layered-API-for-Deep-Learning/
(Section " Incrementally adapting PyTorch code")
data = DataLoaders(train_loader, test_loader).cuda()
learn = Learner(data, Net(), loss_func=F.nll_loss, opt_func=Adam, metrics=accuracy)
learn.fit_one_cycle(epochs, lr)
However, I encountered error --> DataLoader object has no attribute ‘to’dl error 1033×838 49.2 KB
Hi SidNg,
The issue is that both torch and fastai2 define class DataLoader, with the fastai version having extra methods. You need to create your loaders using the fastai2 version of DataLoader. Thendls = DataLoaders(trainDL, validDL).to(device)
HTH, Malcolm
I have 2 torch.utils.data.DataLoader: train_dl and valid_dl
If I do
from fastai.learner import Learner
from fastai.data.core import DataLoaders
from fastai.callback.data import CudaCallback
learn = Learner(dls = DataLoaders(train_dl, valid_dl),
# model, loss, metrics, ...
cbs = [CudaCallback] # And other callbacks
I get : AttributeError: 'DataLoader' object has no attribute 'device'
And if I change to some of this inside the Learner:
dls = DataLoaders(train_dl, valid_dl, device="cuda")dls = DataLoaders(train_dl, valid_dl).cuda()dls = DataLoaders(train_dl, valid_dl).to("cuda")I get : AttributeError: 'DataLoader' object has no attribute 'to'
References
I follow the indications of:
Learner, Metrics, Callbacks – fastai Redirect Thanks in advance for the help
cbs = [CudaCallback] # and others
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-29-b60c6a88c37b> in <module>
----> 1 learn.fit(1, 1e-2) # learn.fit_one_cycle(1, 1e-2)
~/.local/lib/python3.9/site-packages/fastai/learner.py in fit(self, n_epoch, lr, wd, cbs, reset_opt)
210 self.opt.set_hypers(lr=self.lr if lr is None else lr)
211 self.n_epoch = n_epoch
--> 212 self._with_events(self._do_fit, 'fit', CancelFitException, self._end_cleanup)
214 def _end_cleanup(self): self.dl,self.xb,self.yb,self.pred,self.loss = None,(None,),(None,),None,None
~/.local/lib/python3.9/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
159 def _with_events(self, f, event_type, ex, final=noop):
--> 160 try: self(f'before_{event_type}'); f()
161 except ex: self(f'after_cancel_{event_type}')
162 self(f'after_{event_type}'); final()
~/.local/lib/python3.9/site-packages/fastai/learner.py in _do_fit(self)
201 for epoch in range(self.n_epoch):
202 self.epoch=epoch
--> 203 self._with_events(self._do_epoch, 'epoch', CancelEpochException)
205 def fit(self, n_epoch, lr=None, wd=None, cbs=None, reset_opt=False):
~/.local/lib/python3.9/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
159 def _with_events(self, f, event_type, ex, final=noop):
--> 160 try: self(f'before_{event_type}'); f()
161 except ex: self(f'after_cancel_{event_type}')
162 self(f'after_{event_type}'); final()
~/.local/lib/python3.9/site-packages/fastai/learner.py in _do_epoch(self)
196 def _do_epoch(self):
--> 197 self._do_epoch_train()
198 self._do_epoch_validate()
~/.local/lib/python3.9/site-packages/fastai/learner.py in _do_epoch_train(self)
187 def _do_epoch_train(self):
188 self.dl = self.dls.train
--> 189 self._with_events(self.all_batches, 'train', CancelTrainException)
191 def _do_epoch_validate(self, ds_idx=1, dl=None):
~/.local/lib/python3.9/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
159 def _with_events(self, f, event_type, ex, final=noop):
--> 160 try: self(f'before_{event_type}'); f()
161 except ex: self(f'after_cancel_{event_type}')
162 self(f'after_{event_type}'); final()
~/.local/lib/python3.9/site-packages/fastai/learner.py in all_batches(self)
164 def all_batches(self):
165 self.n_iter = len(self.dl)
--> 166 for o in enumerate(self.dl): self.one_batch(*o)
168 def _do_one_batch(self):
~/.local/lib/python3.9/site-packages/fastai/learner.py in one_batch(self, i, b)
181 def one_batch(self, i, b):
182 self.iter = i
--> 183 b_on_device = tuple( e.to(device=self.dls.device) for e in b if hasattr(e, "to")) if self.dls.device is not None else b
184 self._split(b_on_device)
185 self._with_events(self._do_one_batch, 'batch', CancelBatchException)
~/.local/lib/python3.9/site-packages/fastcore/basics.py in __getattr__(self, k)
386 if self._component_attr_filter(k):
387 attr = getattr(self,self._default,None)
--> 388 if attr is not None: return getattr(attr,k)
389 raise AttributeError(k)
390 def __dir__(self): return custom_dir(self,self._dir())
AttributeError: 'DataLoader' object has no attribute 'device'
And this is my error at creating the Learner with on of these options for the dls:
dls = DataLoaders(train_dl, valid_dl, device="cuda")dls = DataLoaders(train_dl, valid_dl).cuda()dls = DataLoaders(train_dl, valid_dl).to("cuda")---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-33-5e8af8ac536d> in <module>
----> 1 learn = Learner(dls = DataLoaders(train_dl, valid_dl, device="cuda"),
2 model = MyModel(),
3 loss_func = CrossEntropyLossFlat())
~/.local/lib/python3.9/site-packages/fastai/data/core.py in __init__(self, path, device, *loaders)
142 def __init__(self, *loaders, path='.', device=None):
143 self.loaders,self.path = list(loaders),Path(path)
--> 144 if device is not None or hasattr(loaders[0],'to'): self.device = device
146 def __getitem__(self, i): return self.loaders[i]
~/.local/lib/python3.9/site-packages/fastai/data/core.py in device(self, d)
158 @device.setter
159 def device(self, d):
--> 160 for dl in self.loaders: dl.to(d)
161 self._device = d
AttributeError: 'DataLoader' object has no attribute 'to'
Actually, this is an active bug. This used to work but something’s changed, let me go dig
I can guarantee this will work with:
!pip install fastai==2.2.5 fastcore==1.3.19
cc @Pomo
Edit: Here’s the breaking commit, will be opening an issue:
github.com/fastai/fastai
Introduced in https://github.com/fastai/fastai/commit/e4303fbb68652a7fe05f28877e… 76a299f3078ec4, the logic there *requires* that your DataLoader's be fastai `DataLoader`'s. As a result using raw torch breaks.
A minimal testing example gist is attached below:
https://gist.github.com/muellerzr/1fb15300ca5f16151af3e0fd2b461614
cc: @jph00
Actually, this is an active bug. This used to work but something’s changed, let me go dig
I can guarantee this will work with:
!pip install fastai==2.2.5 fastcore==1.3.19
Thanks, switching to fastai 2.2.5 meanwhile.
I’ve submitted a PR which reduces what’s needed by you to train your model on the gpu with raw torch loaders, once this get’s merged it’ll be live on the master branch, so install fastai with:
pip install git+https://github.com/fastai/fastai
pip install git+https://github.com/fastai/fastcore
This also will remove the necessary addition of the CudaCallback when training, and instead fastai will just move the tensors to the right device
github.com/fastai/fastai
The original code looked like so:
``… `python
if hasattr(self.dls, "device"): self.model.to(self.dls.device)
And the purpose was to move the model to the GPU or CPU depending on if we're using fastai `DataLoaders`. This isn't particularly flexible though if we're using raw torch loaders. So instead I've opted for the following:
```python
device = getattr(self.dls, 'device', default_device())
self.model.to(device)
This way if we're using raw torch loaders in a CUDA environment, fastai can pick up on that. (More later on how we can still let the user train on the CPU)
## Callback.Data:
Completely removed the `CudaCallback`. With these added changes, it's become irrelevant.
## Learner:
I've added a `_set_device` function to `Learner`. Its goal is to set the batch to the proper device based on the model.
Basic functionality can be summized as if the model and the dataloader's devices are the same, return the DataLoader's device. If not, return the models' device.
This still maintains how fastai determined the model device (as it was all dependent on what the model or DataLoader was set to), but also now allows for raw torch loaders to be set accordingly to the model's device, as this is the only thing they *can* set.
Along with this `one_batch` now calls `_set_device` before `_split`
I've added slight changes to `synth_dbunch` and `synth_learner`. Both now support `tfmdDL=True`. If not, then we will use a pytorch `DataLoader` instead, letting us test and ensure that pytorch `DataLoaders` will *always* be supported within the library, and we can know if anything broke this functionality.
# What This Does to the End Users
Adds a much needed ease of use when using torch `DataLoaders` with fastai's `Learner`, as users would *often* forget (or even know!) that the `CudaCallback` existed, and instead let's them keep the general fastai flow without having to step away from what everyone else using the library is doing.
# Added Tests
As all related `preds` tests are in `Learner`, the added tests for these functionalities also live in there.
## Training with Raw Torch on CPU
The first test is one to ensure we can train with raw pytorch, simply running the same as the training test but also with `tfmdDL=False`:
```python
learn = synth_learner(lr=0.1, tfmdDL=False)
learn(_before_epoch)
learn.model = learn.model.cpu()
xb,yb = next(iter(learn.dls[0]))
init_loss = learn.loss_func(learn.model(xb), yb)
learn.fit(10)
xb,yb = next(iter(learn.dls[0]))
learn.model = learn.model.cpu() # Ensure we're still on CPU even in a CUDA environment
final_loss = learn.loss_func(learn.model(xb), yb)
assert final_loss < init_loss, (final_loss,init_loss)
It should be noted here that to ensure that our model returns as a CPU even in a CUDA environment, we need to set it to CPU again
## Training with Raw Torch on CUDA
To make sure that our device setter is actually working, we run the same small CUDA test but with `tfmdDL=False`:
```python
learn = synth_learner(cbs=TestTrainEvalCallback, tfmdDL=False)
learn.fit(1)
Otherwise all tests were run with both the `cuda` and `slow` flags, and they passed.
cc: @hamelsmu and @jph00
Awesome. Thanks for the PR and making the Learner simpler by removing the necessary addition of the CudaCallback
I have a couple of questions:
The .fit() function of Learner with Pythorch dataloaders works perfectly. But .lr_find() and .fit_one_cycle() dont work.
Error when learn.lr_find():
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-29-fa2b36ab6009> in <module>
----> 1 learn.lr_find()
2 #fig = learn.recorder.plot(suggestion=True, return_fig=True);
3 #lr = learn.recorder.min_grad_lr
~/.local/lib/python3.9/site-packages/fastcore/basics.py in __getattr__(self, k)
386 if self._component_attr_filter(k):
387 attr = getattr(self,self._default,None)
--> 388 if attr is not None: return getattr(attr,k)
389 raise AttributeError(k)
390 def __dir__(self): return custom_dir(self,self._dir())
/usr/lib/python3.9/site-packages/torch/nn/modules/module.py in __getattr__(self, name)
945 if name in modules:
946 return modules[name]
--> 947 raise AttributeError("'{}' object has no attribute '{}'".format(
948 type(self).__name__, name))
AttributeError: 'MyModel' object has no attribute 'lr_find'
Error when learn.fit_one_cycle(1, 1e-2):
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-31-3ea49add0339> in <module>
----> 1 learn.fit_one_cycle(1, 1e-2)
~/.local/lib/python3.9/site-packages/fastcore/basics.py in __getattr__(self, k)
386 if self._component_attr_filter(k):
387 attr = getattr(self,self._default,None)
--> 388 if attr is not None: return getattr(attr,k)
389 raise AttributeError(k)
390 def __dir__(self): return custom_dir(self,self._dir())
/usr/lib/python3.9/site-packages/torch/nn/modules/module.py in __getattr__(self, name)
945 if name in modules:
946 return modules[name]
--> 947 raise AttributeError("'{}' object has no attribute '{}'".format(
948 type(self).__name__, name))
AttributeError: 'MyModel' object has no attribute 'fit_one_cycle'
Maybe for the fit_one_cycle i can solve it by specifiyng schedulers callbacks (ParamScheduler) like the original implementation: https://github.com/fastai/fastai/blob/master/fastai/callback/schedule.py#L110
Thanks, with from fastai.callback.all import * i can use learn.lr_find() and learn.fit_one_cycle(1, 1e-2)
I thought these methods were inside from fastai.learner import Learner
muellerzr: