{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0709bed1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "#| eval: false\n",
    "! [ -e /content ] && pip install -Uqq fastai  # upgrade fastai on colab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6e653c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| default_exp vision.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fb1c331",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "from fastai.torch_basics import *\n",
    "from fastai.data.all import *\n",
    "from fastai.vision.core import *\n",
    "import types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce5d5d36",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "from nbdev.showdoc import *\n",
    "# from fastai.vision.augment import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16b41641",
   "metadata": {},
   "source": [
    "# Vision data\n",
    "\n",
    "> Helper functions to get data in a `DataLoaders` in the vision application and higher class `ImageDataLoaders`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e7518e7",
   "metadata": {},
   "source": [
    "The main classes defined in this module are `ImageDataLoaders` and `SegmentationDataLoaders`, so you probably want to jump to their definitions. They provide factory methods that are a great way to quickly get your data ready for training, see the [vision tutorial](23_tutorial.vision.ipynb) for examples."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc9319c5",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dac27dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@delegates(subplots)\n",
    "def get_grid(\n",
    "    n:int, # Number of axes in the returned grid\n",
    "    nrows:int=None, # Number of rows in the returned grid, defaulting to `int(math.sqrt(n))`\n",
    "    ncols:int=None, # Number of columns in the returned grid, defaulting to `ceil(n/rows)` \n",
    "    figsize:tuple=None, # Width, height in inches of the returned figure\n",
    "    double:bool=False, # Whether to double the number of columns and `n`\n",
    "    title:str=None, # If passed, title set to the figure\n",
    "    return_fig:bool=False, # Whether to return the figure created by `subplots`\n",
    "    flatten:bool=True, # Whether to flatten the matplot axes such that they can be iterated over with a single loop\n",
    "    **kwargs,\n",
    ") -> (plt.Figure, plt.Axes): # Returns just `axs` by default, and (`fig`, `axs`) if `return_fig` is set to True\n",
    "    \"Return a grid of `n` axes, `rows` by `cols`\"\n",
    "    if nrows:\n",
    "        ncols = ncols or int(np.ceil(n/nrows))\n",
    "    elif ncols:\n",
    "        nrows = nrows or int(np.ceil(n/ncols))\n",
    "    else:\n",
    "        nrows = int(math.sqrt(n))\n",
    "        ncols = int(np.ceil(n/nrows))\n",
    "    if double: ncols*=2 ; n*=2\n",
    "    fig,axs = subplots(nrows, ncols, figsize=figsize, **kwargs)\n",
    "    if flatten: axs = [ax if i<n else ax.set_axis_off() for i, ax in enumerate(axs.flatten())][:n]\n",
    "    if title is not None: fig.suptitle(title, weight='bold', size=14)\n",
    "    return (fig,axs) if return_fig else axs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c3cdf94",
   "metadata": {},
   "source": [
    "This is used by the type-dispatched versions of `show_batch` and `show_results` for the vision application. The default `figsize` is `(cols*imsize, rows*imsize+0.6)`. `imsize` is passed down to `subplots`. `suptitle`, `sharex`, `sharey`, `squeeze`, `subplot_kw` and `gridspec_kw` are all passed down to [plt.subplots](https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.subplots.html#matplotlib-pyplot-subplots). If `return_fig` is `True`, returns `fig,axs`, otherwise just `axs`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90ab8121",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def clip_remove_empty(\n",
    "    bbox:TensorBBox, # Coordinates of bounding boxes \n",
    "    label:TensorMultiCategory # Labels of the bounding boxes\n",
    "):\n",
    "    \"Clip bounding boxes with image border and remove empty boxes along with corresponding labels\"\n",
    "    bbox = torch.clamp(bbox, -1, 1)\n",
    "    empty = ((bbox[...,2] - bbox[...,0])*(bbox[...,3] - bbox[...,1]) <= 0.)\n",
    "    return (bbox[~empty], label[TensorBase(~empty)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85601fe1",
   "metadata": {},
   "source": [
    "This is used in `bb_pad`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73e05130",
   "metadata": {},
   "outputs": [],
   "source": [
    "bb = TensorBBox([[-2,-0.5,0.5,1.5], [-0.5,-0.5,0.5,0.5], [1,0.5,0.5,0.75], [-0.5,-0.5,0.5,0.5], [-2, -0.5, -1.5, 0.5]])\n",
    "bb,lbl = clip_remove_empty(bb, TensorMultiCategory([1,2,3,2,5]))\n",
    "test_eq(bb, TensorBBox([[-1,-0.5,0.5,1.], [-0.5,-0.5,0.5,0.5], [-0.5,-0.5,0.5,0.5]]))\n",
    "test_eq(lbl, TensorMultiCategory([1,2,2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4bd0c09",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def bb_pad(\n",
    "    samples:list, # List of 3-tuples like (image, bounding_boxes, labels)\n",
    "    pad_idx=0 # Label that will be used to pad each list of labels\n",
    "):\n",
    "    \"Function that collects `samples` of labelled bboxes and adds padding with `pad_idx`.\"\n",
    "    samples = [(s[0], *clip_remove_empty(*s[1:])) for s in samples]\n",
    "    max_len = max([len(s[2]) for s in samples])\n",
    "    def _f(img,bbox,lbl):\n",
    "        bbox = torch.cat([bbox,bbox.new_zeros(max_len-bbox.shape[0], 4)])\n",
    "        lbl  = torch.cat([lbl, lbl .new_zeros(max_len-lbl .shape[0])+pad_idx])\n",
    "        return img,bbox,lbl\n",
    "    return [_f(*s) for s in samples]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "664f8be8",
   "metadata": {},
   "source": [
    "This is used in `BBoxBlock`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41c0cec3",
   "metadata": {},
   "outputs": [],
   "source": [
    "img1,img2 = TensorImage(torch.randn(16,16,3)),TensorImage(torch.randn(16,16,3))\n",
    "bb1 = tensor([[-2,-0.5,0.5,1.5], [-0.5,-0.5,0.5,0.5], [1,0.5,0.5,0.75], [-0.5,-0.5,0.5,0.5]])\n",
    "lbl1 = tensor([1, 2, 3, 2])\n",
    "bb2 = tensor([[-0.5,-0.5,0.5,0.5], [-0.5,-0.5,0.5,0.5]])\n",
    "lbl2 = tensor([2, 2])\n",
    "samples = [(img1, bb1, lbl1), (img2, bb2, lbl2)]\n",
    "res = bb_pad(samples)\n",
    "non_empty = tensor([True,True,False,True])\n",
    "test_eq(res[0][0], img1)\n",
    "test_eq(res[0][1], tensor([[-1,-0.5,0.5,1.], [-0.5,-0.5,0.5,0.5], [-0.5,-0.5,0.5,0.5]]))\n",
    "test_eq(res[0][2], tensor([1,2,2]))\n",
    "test_eq(res[1][0], img2)\n",
    "test_eq(res[1][1], tensor([[-0.5,-0.5,0.5,0.5], [-0.5,-0.5,0.5,0.5], [0,0,0,0]]))\n",
    "test_eq(res[1][2], tensor([2,2,0]))      "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fd28528",
   "metadata": {},
   "source": [
    "## Show methods -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da32c0df",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@dispatch\n",
    "def show_batch(x:TensorImage, y, samples, ctxs=None, max_n=10, nrows=None, ncols=None, figsize=None, **kwargs):\n",
    "    if ctxs is None: ctxs = get_grid(min(len(samples), max_n), nrows=nrows, ncols=ncols, figsize=figsize)\n",
    "    return get_show_batch_func(object)(x, y, samples, ctxs=ctxs, max_n=max_n, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8fa5be49",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@dispatch\n",
    "def show_batch(x:TensorImage, y:TensorImage, samples, ctxs=None, max_n=10, nrows=None, ncols=None, figsize=None, **kwargs):\n",
    "    if ctxs is None: ctxs = get_grid(min(len(samples), max_n), nrows=nrows, ncols=ncols, figsize=figsize, double=True)\n",
    "    for i in range(2):\n",
    "        ctxs[i::2] = [b.show(ctx=c, **kwargs) for b,c,_ in zip(samples.itemgot(i),ctxs[i::2],range(max_n))]\n",
    "    return ctxs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94e38b04",
   "metadata": {},
   "source": [
    "## `TransformBlock`s for vision"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a66c4ac",
   "metadata": {},
   "source": [
    "These are the blocks the vision application provide for the [data block API](06_data.block.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d076082a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def ImageBlock(cls:PILBase=PILImage):\n",
    "    \"A `TransformBlock` for images of `cls`\"\n",
    "    return TransformBlock(type_tfms=cls.create, batch_tfms=IntToFloatTensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0443e39",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def MaskBlock(\n",
    "    codes:list=None # Vocab labels for segmentation masks\n",
    "):\n",
    "    \"A `TransformBlock` for segmentation masks, potentially with `codes`\"\n",
    "    return TransformBlock(type_tfms=PILMask.create, item_tfms=AddMaskCodes(codes=codes), batch_tfms=IntToFloatTensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba78d51b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "\n",
       "```python\n",
       "class TransformBlock():\n",
       "    \"A basic wrapper that links defaults transforms for the data block API\"\n",
       "    def __init__(self, \n",
       "        type_tfms:list=None, # One or more `Transform`s\n",
       "        item_tfms:list=None, # `ItemTransform`s, applied on an item\n",
       "        batch_tfms:list=None, # `Transform`s or `RandTransform`s, applied by batch\n",
       "        dl_type:TfmdDL=None, # Task specific `TfmdDL`, defaults to `TfmdDL`\n",
       "        dls_kwargs:dict=None, # Additional arguments to be passed to `DataLoaders`\n",
       "    ):\n",
       "        self.type_tfms  =            L(type_tfms)\n",
       "        self.item_tfms  = ToTensor + L(item_tfms)\n",
       "        self.batch_tfms =            L(batch_tfms)\n",
       "        self.dl_type,self.dls_kwargs = dl_type,({} if dls_kwargs is None else dls_kwargs)\n",
       "```\n",
       "\n",
       "**File:** `~/aai-ws/fastai/fastai/data/block.py`"
      ],
      "text/plain": [
       "\u001b[31mInit signature:\u001b[39m\n",
       "TransformBlock(\n",
       "    type_tfms: \u001b[33m'list'\u001b[39m = \u001b[38;5;28;01mNone\u001b[39;00m,\n",
       "    item_tfms: \u001b[33m'list'\u001b[39m = \u001b[38;5;28;01mNone\u001b[39;00m,\n",
       "    batch_tfms: \u001b[33m'list'\u001b[39m = \u001b[38;5;28;01mNone\u001b[39;00m,\n",
       "    dl_type: \u001b[33m'TfmdDL'\u001b[39m = \u001b[38;5;28;01mNone\u001b[39;00m,\n",
       "    dls_kwargs: \u001b[33m'dict'\u001b[39m = \u001b[38;5;28;01mNone\u001b[39;00m,\n",
       ")\n",
       "\u001b[31mSource:\u001b[39m        \n",
       "\u001b[38;5;28;01mclass\u001b[39;00m TransformBlock():\n",
       "    \u001b[33m\"A basic wrapper that links defaults transforms for the data block API\"\u001b[39m\n",
       "    \u001b[38;5;28;01mdef\u001b[39;00m __init__(self, \n",
       "        type_tfms:list=\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;66;03m# One or more `Transform`s\u001b[39;00m\n",
       "        item_tfms:list=\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;66;03m# `ItemTransform`s, applied on an item\u001b[39;00m\n",
       "        batch_tfms:list=\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;66;03m# `Transform`s or `RandTransform`s, applied by batch\u001b[39;00m\n",
       "        dl_type:TfmdDL=\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;66;03m# Task specific `TfmdDL`, defaults to `TfmdDL`\u001b[39;00m\n",
       "        dls_kwargs:dict=\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;66;03m# Additional arguments to be passed to `DataLoaders`\u001b[39;00m\n",
       "    ):\n",
       "        self.type_tfms  =            L(type_tfms)\n",
       "        self.item_tfms  = ToTensor + L(item_tfms)\n",
       "        self.batch_tfms =            L(batch_tfms)\n",
       "        self.dl_type,self.dls_kwargs = dl_type,({} \u001b[38;5;28;01mif\u001b[39;00m dls_kwargs \u001b[38;5;28;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m dls_kwargs)\n",
       "\u001b[31mFile:\u001b[39m           ~/aai-ws/fastai/fastai/data/block.py\n",
       "\u001b[31mType:\u001b[39m           type\n",
       "\u001b[31mSubclasses:\u001b[39m     "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "TransformBlock??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a844ce5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "PointBlock = TransformBlock(type_tfms=TensorPoint.create, item_tfms=PointScaler)\n",
    "BBoxBlock = TransformBlock(type_tfms=TensorBBox.create, item_tfms=PointScaler, dls_kwargs = {'before_batch': bb_pad})\n",
    "\n",
    "PointBlock.__doc__ = \"A `TransformBlock` for points in an image\"\n",
    "BBoxBlock.__doc__  = \"A `TransformBlock` for bounding boxes in an image\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e54a03be",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def BBoxLblBlock(\n",
    "    vocab:list=None, # Vocab labels for bounding boxes\n",
    "    add_na:bool=True # Add NaN as a background class\n",
    "):\n",
    "    \"A `TransformBlock` for labeled bounding boxes, potentially with `vocab`\"\n",
    "    return TransformBlock(type_tfms=MultiCategorize(vocab=vocab, add_na=add_na), item_tfms=BBoxLabeler)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7aeff9a2",
   "metadata": {},
   "source": [
    "If `add_na` is `True`, a new category is added for NaN (that will represent the background class)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71e50424",
   "metadata": {},
   "source": [
    "## ImageDataLoaders -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4e7db9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "class ImageDataLoaders(DataLoaders):\n",
    "    \"Basic wrapper around several `DataLoader`s with factory methods for computer vision problems\"\n",
    "    @classmethod\n",
    "    @delegates(DataLoaders.from_dblock)\n",
    "    def from_folder(cls, path, train='train', valid='valid', valid_pct=None, seed=None, vocab=None, item_tfms=None,\n",
    "                    batch_tfms=None, img_cls=PILImage, **kwargs):\n",
    "        \"Create from imagenet style dataset in `path` with `train` and `valid` subfolders (or provide `valid_pct`)\"\n",
    "        splitter = GrandparentSplitter(train_name=train, valid_name=valid) if valid_pct is None else RandomSplitter(valid_pct, seed=seed)\n",
    "        get_items = get_image_files if valid_pct else partial(get_image_files, folders=[train, valid])\n",
    "        dblock = DataBlock(blocks=(ImageBlock(img_cls), CategoryBlock(vocab=vocab)),\n",
    "                           get_items=get_items,\n",
    "                           splitter=splitter,\n",
    "                           get_y=parent_label,\n",
    "                           item_tfms=item_tfms,\n",
    "                           batch_tfms=batch_tfms)\n",
    "        return cls.from_dblock(dblock, path, path=path, **kwargs)\n",
    "\n",
    "    @classmethod\n",
    "    @delegates(DataLoaders.from_dblock)\n",
    "    def from_path_func(cls, path, fnames, label_func, valid_pct=0.2, seed=None, item_tfms=None, batch_tfms=None, \n",
    "                       img_cls=PILImage, **kwargs):\n",
    "        \"Create from list of `fnames` in `path`s with `label_func`\"\n",
    "        dblock = DataBlock(blocks=(ImageBlock(img_cls), CategoryBlock),\n",
    "                           splitter=RandomSplitter(valid_pct, seed=seed),\n",
    "                           get_y=label_func,\n",
    "                           item_tfms=item_tfms,\n",
    "                           batch_tfms=batch_tfms)\n",
    "        return cls.from_dblock(dblock, fnames, path=path, **kwargs)\n",
    "\n",
    "    @classmethod\n",
    "    def from_name_func(cls,\n",
    "        path:str|Path, # Set the default path to a directory that a `Learner` can use to save files like models\n",
    "        fnames:list, # A list of `os.Pathlike`'s to individual image files\n",
    "        label_func:Callable, # A function that receives a string (the file name) and outputs a label\n",
    "        **kwargs\n",
    "    ) -> DataLoaders:\n",
    "        \"Create from the name attrs of `fnames` in `path`s with `label_func`\"\n",
    "        if sys.platform == 'win32' and isinstance(label_func, types.LambdaType) and label_func.__name__ == '<lambda>':\n",
    "            # https://medium.com/@jwnx/multiprocessing-serialization-in-python-with-pickle-9844f6fa1812\n",
    "            raise ValueError(\"label_func couldn't be lambda function on Windows\")\n",
    "        f = using_attr(label_func, 'name')\n",
    "        return cls.from_path_func(path, fnames, f, **kwargs)\n",
    "\n",
    "    @classmethod\n",
    "    def from_path_re(cls, path, fnames, pat, **kwargs):\n",
    "        \"Create from list of `fnames` in `path`s with re expression `pat`\"\n",
    "        return cls.from_path_func(path, fnames, RegexLabeller(pat), **kwargs)\n",
    "\n",
    "    @classmethod\n",
    "    @delegates(DataLoaders.from_dblock)\n",
    "    def from_name_re(cls, path, fnames, pat, **kwargs):\n",
    "        \"Create from the name attrs of `fnames` in `path`s with re expression `pat`\"\n",
    "        return cls.from_name_func(path, fnames, RegexLabeller(pat), **kwargs)\n",
    "\n",
    "    @classmethod\n",
    "    @delegates(DataLoaders.from_dblock)\n",
    "    def from_df(cls, df, path='.', valid_pct=0.2, seed=None, fn_col=0, folder=None, suff='', label_col=1, label_delim=None,\n",
    "                y_block=None, valid_col=None, item_tfms=None, batch_tfms=None, img_cls=PILImage, **kwargs):\n",
    "        \"Create from `df` using `fn_col` and `label_col`\"\n",
    "        pref = f'{Path(path) if folder is None else Path(path)/folder}{os.path.sep}'\n",
    "        if y_block is None:\n",
    "            is_multi = (is_listy(label_col) and len(label_col) > 1) or label_delim is not None\n",
    "            y_block = MultiCategoryBlock if is_multi else CategoryBlock\n",
    "        splitter = RandomSplitter(valid_pct, seed=seed) if valid_col is None else ColSplitter(valid_col)\n",
    "        dblock = DataBlock(blocks=(ImageBlock(img_cls), y_block),\n",
    "                           get_x=ColReader(fn_col, pref=pref, suff=suff),\n",
    "                           get_y=ColReader(label_col, label_delim=label_delim),\n",
    "                           splitter=splitter,\n",
    "                           item_tfms=item_tfms,\n",
    "                           batch_tfms=batch_tfms)\n",
    "        return cls.from_dblock(dblock, df, path=path, **kwargs)\n",
    "\n",
    "    @classmethod\n",
    "    def from_csv(cls, path, csv_fname='labels.csv', header='infer', delimiter=None, quoting=csv.QUOTE_MINIMAL, **kwargs):\n",
    "        \"Create from `path/csv_fname` using `fn_col` and `label_col`\"\n",
    "        df = pd.read_csv(Path(path)/csv_fname, header=header, delimiter=delimiter, quoting=quoting)\n",
    "        return cls.from_df(df, path=path, **kwargs)\n",
    "\n",
    "    @classmethod\n",
    "    @delegates(DataLoaders.from_dblock)\n",
    "    def from_lists(cls, path, fnames, labels, valid_pct=0.2, seed:int=None, y_block=None, item_tfms=None, batch_tfms=None,\n",
    "                   img_cls=PILImage, **kwargs):\n",
    "        \"Create from list of `fnames` and `labels` in `path`\"\n",
    "        if y_block is None:\n",
    "            y_block = MultiCategoryBlock if is_listy(labels[0]) and len(labels[0]) > 1 else (\n",
    "                RegressionBlock if isinstance(labels[0], float) else CategoryBlock)\n",
    "        dblock = DataBlock.from_columns(blocks=(ImageBlock(img_cls), y_block),\n",
    "                           splitter=RandomSplitter(valid_pct, seed=seed),\n",
    "                           item_tfms=item_tfms,\n",
    "                           batch_tfms=batch_tfms)\n",
    "        return cls.from_dblock(dblock, (fnames, labels), path=path, **kwargs)\n",
    "\n",
    "ImageDataLoaders.from_csv = delegates(to=ImageDataLoaders.from_df)(ImageDataLoaders.from_csv)\n",
    "ImageDataLoaders.from_name_func = delegates(to=ImageDataLoaders.from_path_func)(ImageDataLoaders.from_name_func)\n",
    "ImageDataLoaders.from_path_re = delegates(to=ImageDataLoaders.from_path_func)(ImageDataLoaders.from_path_re)\n",
    "ImageDataLoaders.from_name_re = delegates(to=ImageDataLoaders.from_name_func)(ImageDataLoaders.from_name_re)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c354fb9",
   "metadata": {},
   "source": [
    "This class should not be used directly, one of the factory methods should be preferred instead. All those factory methods accept as arguments:\n",
    "\n",
    "- `item_tfms`: one or several transforms applied to the items before batching them\n",
    "- `batch_tfms`: one or several transforms applied to the batches once they are formed\n",
    "- `bs`: the batch size\n",
    "- `val_bs`: the batch size for the validation `DataLoader` (defaults to `bs`)\n",
    "- `shuffle_train`: if we shuffle the training `DataLoader` or not\n",
    "- `device`: the PyTorch device to use (defaults to `default_device()`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dba64c93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/fastai/fastai/blob/main/fastai/vision/data.py#L112){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### ImageDataLoaders.from_folder\n",
       "\n",
       "```python\n",
       "\n",
       "def from_folder(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    train:str='train', valid:str='valid', valid_pct:NoneType=None, seed:NoneType=None, vocab:NoneType=None,\n",
       "    item_tfms:NoneType=None, batch_tfms:NoneType=None, img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from imagenet style dataset in `path` with `train` and `valid` subfolders (or provide `valid_pct`)*"
      ],
      "text/plain": [
       "```python\n",
       "\n",
       "def from_folder(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    train:str='train', valid:str='valid', valid_pct:NoneType=None, seed:NoneType=None, vocab:NoneType=None,\n",
       "    item_tfms:NoneType=None, batch_tfms:NoneType=None, img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from imagenet style dataset in `path` with `train` and `valid` subfolders (or provide `valid_pct`)*"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(ImageDataLoaders.from_folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25191e6e",
   "metadata": {},
   "source": [
    "If `valid_pct` is provided, a random split is performed (with an optional `seed`) by setting aside that percentage of the data for the validation set (instead of looking at the grandparents folder). If a `vocab` is passed, only the folders with names in `vocab` are kept.\n",
    "\n",
    "Here is an example loading a subsample of MNIST:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3896b42",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = untar_data(URLs.MNIST_TINY)\n",
    "dls = ImageDataLoaders.from_folder(path, img_cls=PILImageBW)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfab9bf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "x,y = dls.one_batch()\n",
    "test_eq(x.shape, [64, 1, 28, 28])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ad48183",
   "metadata": {},
   "source": [
    "Passing `valid_pct` will ignore the valid/train folders and do a new random split:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "964e8b69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Path('/Users/jhoward/.fastai/data/mnist_tiny/valid/7/9919.png'),\n",
       " Path('/Users/jhoward/.fastai/data/mnist_tiny/train/3/9637.png'),\n",
       " Path('/Users/jhoward/.fastai/data/mnist_tiny/valid/3/7407.png')]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dls = ImageDataLoaders.from_folder(path, valid_pct=0.2)\n",
    "dls.valid_ds.items[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed7b0fd6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/fastai/fastai/blob/main/fastai/vision/data.py#L127){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### ImageDataLoaders.from_path_func\n",
       "\n",
       "```python\n",
       "\n",
       "def from_path_func(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    fnames, label_func, valid_pct:float=0.2, seed:NoneType=None, item_tfms:NoneType=None, batch_tfms:NoneType=None,\n",
       "    img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from list of `fnames` in `path`s with `label_func`*"
      ],
      "text/plain": [
       "```python\n",
       "\n",
       "def from_path_func(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    fnames, label_func, valid_pct:float=0.2, seed:NoneType=None, item_tfms:NoneType=None, batch_tfms:NoneType=None,\n",
       "    img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from list of `fnames` in `path`s with `label_func`*"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(ImageDataLoaders.from_path_func)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9150e006",
   "metadata": {},
   "source": [
    "The validation set is a random `subset` of `valid_pct`, optionally created with `seed` for reproducibility.\n",
    "\n",
    "Here is how to create the same `DataLoaders` on the MNIST dataset as the previous example with a `label_func`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "100c2fcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "fnames = get_image_files(path)\n",
    "def label_func(x): return x.parent.name\n",
    "dls = ImageDataLoaders.from_path_func(path, fnames, label_func)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abe4e911",
   "metadata": {},
   "source": [
    "Here is another example on the pets dataset. Here filenames are all in an \"images\" folder and their names have the form `class_name_123.jpg`. One way to properly label them is thus to throw away everything after the last `_`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a5d054b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/fastai/fastai/blob/main/fastai/vision/data.py#L152){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### ImageDataLoaders.from_path_re\n",
       "\n",
       "```python\n",
       "\n",
       "def from_path_re(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    fnames, pat, valid_pct:float=0.2, seed:NoneType=None, item_tfms:NoneType=None, batch_tfms:NoneType=None,\n",
       "    img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from list of `fnames` in `path`s with re expression `pat`*"
      ],
      "text/plain": [
       "```python\n",
       "\n",
       "def from_path_re(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    fnames, pat, valid_pct:float=0.2, seed:NoneType=None, item_tfms:NoneType=None, batch_tfms:NoneType=None,\n",
       "    img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from list of `fnames` in `path`s with re expression `pat`*"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(ImageDataLoaders.from_path_re)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88280857",
   "metadata": {},
   "source": [
    "The validation set is a random subset of `valid_pct`, optionally created with `seed` for reproducibility.\n",
    "\n",
    "Here is how to create the same `DataLoaders` on the MNIST dataset as the previous example (you will need to change the initial two / by a \\ on Windows):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bc54ace",
   "metadata": {},
   "outputs": [],
   "source": [
    "pat = r'/([^/]*)/\\d+.png$'\n",
    "dls = ImageDataLoaders.from_path_re(path, fnames, pat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d000efd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/fastai/fastai/blob/main/fastai/vision/data.py#L138){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### ImageDataLoaders.from_name_func\n",
       "\n",
       "```python\n",
       "\n",
       "def from_name_func(\n",
       "    path:str | pathlib.Path, # Set the default path to a directory that a `Learner` can use to save files like models\n",
       "    fnames:list, # A list of `os.Pathlike`'s to individual image files\n",
       "    label_func:Callable, # A function that receives a string (the file name) and outputs a label\n",
       "    valid_pct:float=0.2, seed:NoneType=None, item_tfms:NoneType=None, batch_tfms:NoneType=None,\n",
       "    img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       ")->DataLoaders:\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from the name attrs of `fnames` in `path`s with `label_func`*"
      ],
      "text/plain": [
       "```python\n",
       "\n",
       "def from_name_func(\n",
       "    path:str | pathlib.Path, # Set the default path to a directory that a `Learner` can use to save files like models\n",
       "    fnames:list, # A list of `os.Pathlike`'s to individual image files\n",
       "    label_func:Callable, # A function that receives a string (the file name) and outputs a label\n",
       "    valid_pct:float=0.2, seed:NoneType=None, item_tfms:NoneType=None, batch_tfms:NoneType=None,\n",
       "    img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       ")->DataLoaders:\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from the name attrs of `fnames` in `path`s with `label_func`*"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(ImageDataLoaders.from_name_func)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "105cfbe0",
   "metadata": {},
   "source": [
    "The validation set is a random subset of `valid_pct`, optionally created with `seed` for reproducibility. This method does the same as `ImageDataLoaders.from_path_func` except `label_func` is applied to the name of each filenames, and not the full path."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "390b2187",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/fastai/fastai/blob/main/fastai/vision/data.py#L158){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### ImageDataLoaders.from_name_re\n",
       "\n",
       "```python\n",
       "\n",
       "def from_name_re(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    fnames, pat, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from the name attrs of `fnames` in `path`s with re expression `pat`*"
      ],
      "text/plain": [
       "```python\n",
       "\n",
       "def from_name_re(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    fnames, pat, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from the name attrs of `fnames` in `path`s with re expression `pat`*"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(ImageDataLoaders.from_name_re)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a8259ff",
   "metadata": {},
   "source": [
    "The validation set is a random subset of `valid_pct`, optionally created with `seed` for reproducibility. This method does the same as `ImageDataLoaders.from_path_re` except `pat` is applied to the name of each filenames, and not the full path."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98e86d3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/fastai/fastai/blob/main/fastai/vision/data.py#L164){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### ImageDataLoaders.from_df\n",
       "\n",
       "```python\n",
       "\n",
       "def from_df(\n",
       "    df, path:str='.', # Path to put in `DataLoaders`\n",
       "    valid_pct:float=0.2, seed:NoneType=None, fn_col:int=0, folder:NoneType=None, suff:str='', label_col:int=1,\n",
       "    label_delim:NoneType=None, y_block:NoneType=None, valid_col:NoneType=None, item_tfms:NoneType=None,\n",
       "    batch_tfms:NoneType=None, img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from `df` using `fn_col` and `label_col`*"
      ],
      "text/plain": [
       "```python\n",
       "\n",
       "def from_df(\n",
       "    df, path:str='.', # Path to put in `DataLoaders`\n",
       "    valid_pct:float=0.2, seed:NoneType=None, fn_col:int=0, folder:NoneType=None, suff:str='', label_col:int=1,\n",
       "    label_delim:NoneType=None, y_block:NoneType=None, valid_col:NoneType=None, item_tfms:NoneType=None,\n",
       "    batch_tfms:NoneType=None, img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from `df` using `fn_col` and `label_col`*"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(ImageDataLoaders.from_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "650192c5",
   "metadata": {},
   "source": [
    "The validation set is a random subset of `valid_pct`, optionally created with `seed` for reproducibility. Alternatively, if your `df` contains a `valid_col`, give its name or its index to that argument (the column should have `True` for the elements going to the validation set). \n",
    "\n",
    "You can add an additional `folder` to the filenames in `df` if they should not be concatenated directly to `path`. If they do not contain the proper extensions, you can add `suff`. If your label column contains multiple labels on each row, you can use `label_delim` to warn the library you have a multi-label problem. \n",
    "\n",
    "`y_block` should be passed when the task automatically picked by the library is wrong, you should then give `CategoryBlock`, `MultiCategoryBlock` or `RegressionBlock`. For more advanced uses, you should use the data block API.\n",
    "\n",
    "The tiny mnist example from before also contains a version in a dataframe:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3927a52a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train/3/7463.png</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train/3/9829.png</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train/3/7881.png</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train/3/8065.png</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train/3/7046.png</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               name  label\n",
       "0  train/3/7463.png      3\n",
       "1  train/3/9829.png      3\n",
       "2  train/3/7881.png      3\n",
       "3  train/3/8065.png      3\n",
       "4  train/3/7046.png      3"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = untar_data(URLs.MNIST_TINY)\n",
    "df = pd.read_csv(path/'labels.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5b278ec",
   "metadata": {},
   "source": [
    "Here is how to load it using `ImageDataLoaders.from_df`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "123db1df",
   "metadata": {},
   "outputs": [],
   "source": [
    "dls = ImageDataLoaders.from_df(df, path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff9a83bb",
   "metadata": {},
   "source": [
    "Here is another example with a multi-label problem:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c24d5c2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fname</th>\n",
       "      <th>labels</th>\n",
       "      <th>is_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000005.jpg</td>\n",
       "      <td>chair</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000007.jpg</td>\n",
       "      <td>car</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000009.jpg</td>\n",
       "      <td>horse person</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000012.jpg</td>\n",
       "      <td>car</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000016.jpg</td>\n",
       "      <td>bicycle</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        fname        labels  is_valid\n",
       "0  000005.jpg         chair      True\n",
       "1  000007.jpg           car      True\n",
       "2  000009.jpg  horse person      True\n",
       "3  000012.jpg           car     False\n",
       "4  000016.jpg       bicycle      True"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| eval: false\n",
    "path = untar_data(URLs.PASCAL_2007)\n",
    "df = pd.read_csv(path/'train.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2093771",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| eval: false\n",
    "dls = ImageDataLoaders.from_df(df, path, folder='train', valid_col='is_valid')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1772b9d",
   "metadata": {},
   "source": [
    "Note that can also pass `2` to valid_col (the index, starting with 0)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1431ba24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/fastai/fastai/blob/main/fastai/vision/data.py#L181){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### ImageDataLoaders.from_csv\n",
       "\n",
       "```python\n",
       "\n",
       "def from_csv(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    csv_fname:str='labels.csv', header:str='infer', delimiter:NoneType=None, quoting:int=0, valid_pct:float=0.2,\n",
       "    seed:NoneType=None, fn_col:int=0, folder:NoneType=None, suff:str='', label_col:int=1, label_delim:NoneType=None,\n",
       "    y_block:NoneType=None, valid_col:NoneType=None, item_tfms:NoneType=None, batch_tfms:NoneType=None,\n",
       "    img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from `path/csv_fname` using `fn_col` and `label_col`*"
      ],
      "text/plain": [
       "```python\n",
       "\n",
       "def from_csv(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    csv_fname:str='labels.csv', header:str='infer', delimiter:NoneType=None, quoting:int=0, valid_pct:float=0.2,\n",
       "    seed:NoneType=None, fn_col:int=0, folder:NoneType=None, suff:str='', label_col:int=1, label_delim:NoneType=None,\n",
       "    y_block:NoneType=None, valid_col:NoneType=None, item_tfms:NoneType=None, batch_tfms:NoneType=None,\n",
       "    img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from `path/csv_fname` using `fn_col` and `label_col`*"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(ImageDataLoaders.from_csv)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f97fbf42",
   "metadata": {},
   "source": [
    "Same as `ImageDataLoaders.from_df` after loading the file with `header` and `delimiter`.\n",
    "\n",
    "Here is how to load the same dataset as before with this method:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75d4a6bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| eval: false\n",
    "dls = ImageDataLoaders.from_csv(path, 'train.csv', folder='train', valid_col='is_valid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "949e0bca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/fastai/fastai/blob/main/fastai/vision/data.py#L188){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### ImageDataLoaders.from_lists\n",
       "\n",
       "```python\n",
       "\n",
       "def from_lists(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    fnames, labels, valid_pct:float=0.2, seed:int=None, y_block:NoneType=None, item_tfms:NoneType=None,\n",
       "    batch_tfms:NoneType=None, img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from list of `fnames` and `labels` in `path`*"
      ],
      "text/plain": [
       "```python\n",
       "\n",
       "def from_lists(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    fnames, labels, valid_pct:float=0.2, seed:int=None, y_block:NoneType=None, item_tfms:NoneType=None,\n",
       "    batch_tfms:NoneType=None, img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from list of `fnames` and `labels` in `path`*"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(ImageDataLoaders.from_lists)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14a54a6a",
   "metadata": {},
   "source": [
    "The validation set is a random subset of `valid_pct`, optionally created with `seed` for reproducibility. `y_block` can be passed to specify the type of the targets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf31c5a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = untar_data(URLs.PETS)\n",
    "fnames = get_image_files(path/\"images\")\n",
    "labels = ['_'.join(x.name.split('_')[:-1]) for x in fnames]\n",
    "dls = ImageDataLoaders.from_lists(path, fnames, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f651786a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "class SegmentationDataLoaders(DataLoaders):\n",
    "    \"Basic wrapper around several `DataLoader`s with factory methods for segmentation problems\"\n",
    "    @classmethod\n",
    "    @delegates(DataLoaders.from_dblock)\n",
    "    def from_label_func(cls, path, fnames, label_func, valid_pct=0.2, seed=None, codes=None, item_tfms=None, batch_tfms=None, \n",
    "                        img_cls=PILImage, **kwargs):\n",
    "        \"Create from list of `fnames` in `path`s with `label_func`.\"\n",
    "        dblock = DataBlock(blocks=(ImageBlock(img_cls), MaskBlock(codes=codes)),\n",
    "                           splitter=RandomSplitter(valid_pct, seed=seed),\n",
    "                           get_y=label_func,\n",
    "                           item_tfms=item_tfms,\n",
    "                           batch_tfms=batch_tfms)\n",
    "        res = cls.from_dblock(dblock, fnames, path=path, **kwargs)\n",
    "        return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59ddb610",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/fastai/fastai/blob/main/fastai/vision/data.py#L210){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### SegmentationDataLoaders.from_label_func\n",
       "\n",
       "```python\n",
       "\n",
       "def from_label_func(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    fnames, label_func, valid_pct:float=0.2, seed:NoneType=None, codes:NoneType=None, item_tfms:NoneType=None,\n",
       "    batch_tfms:NoneType=None, img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from list of `fnames` in `path`s with `label_func`.*"
      ],
      "text/plain": [
       "```python\n",
       "\n",
       "def from_label_func(\n",
       "    path, # Path to put in `DataLoaders`\n",
       "    fnames, label_func, valid_pct:float=0.2, seed:NoneType=None, codes:NoneType=None, item_tfms:NoneType=None,\n",
       "    batch_tfms:NoneType=None, img_cls:BypassNewMeta=PILImage, bs:int=64, # Size of batch\n",
       "    val_bs:int=None, # Size of batch for validation `DataLoader`\n",
       "    shuffle:bool=True, # Whether to shuffle data\n",
       "    device:NoneType=None, # Device to put `DataLoaders`\n",
       "):\n",
       "\n",
       "\n",
       "```\n",
       "\n",
       "*Create from list of `fnames` in `path`s with `label_func`.*"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(SegmentationDataLoaders.from_label_func)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "646da74a",
   "metadata": {},
   "source": [
    "The validation set is a random subset of `valid_pct`, optionally created with `seed` for reproducibility. `codes` contain the mapping index to label."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "345605c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = untar_data(URLs.CAMVID_TINY)\n",
    "fnames = get_image_files(path/'images')\n",
    "def label_func(x): return path/'labels'/f'{x.stem}_P{x.suffix}'\n",
    "codes = np.loadtxt(path/'codes.txt', dtype=str)\n",
    "    \n",
    "dls = SegmentationDataLoaders.from_label_func(path, fnames, label_func, codes=codes)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56702f65",
   "metadata": {},
   "source": [
    "# Export -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5af1113d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "from nbdev import nbdev_export\n",
    "nbdev_export()"
   ]
  }
 ],
 "metadata": {},
 "nbformat": 4,
 "nbformat_minor": 5
}