{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NTaMBjChz7m5"
      },
      "source": [
        "# Hugging Face Datasets to Opik Integration\n",
        "\n",
        "You will need:\n",
        "\n",
        "1. A Comet account, for seeing Opik visualizations (free!) - [comet.com](https://comet.com)\n",
        "2. Optional: A Hugging Face account for private datasets - [huggingface.co/settings/tokens](https://huggingface.co/settings/tokens)\n",
        "\n",
        "This example will use:\n",
        "\n",
        "- [datasets](https://pypi.org/project/datasets/) to load Hugging Face datasets\n",
        "- [opik](https://pypi.org/project/opik/) to create and manage Opik datasets"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yM1lU0dBBnJs"
      },
      "source": [
        "## Setup\n",
        "\n",
        "This pip-install takes about a minute."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "2Tx6HwuU1rB4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: opik in /home/mavrick/anaconda3/lib/python3.12/site-packages (1.7.36)\n",
            "Requirement already satisfied: datasets in /home/mavrick/anaconda3/lib/python3.12/site-packages (3.6.0)\n",
            "Requirement already satisfied: transformers in /home/mavrick/anaconda3/lib/python3.12/site-packages (4.52.4)\n",
            "Requirement already satisfied: pandas in /home/mavrick/anaconda3/lib/python3.12/site-packages (2.3.0)\n",
            "Requirement already satisfied: tqdm in /home/mavrick/anaconda3/lib/python3.12/site-packages (4.67.1)\n",
            "Requirement already satisfied: huggingface_hub in /home/mavrick/anaconda3/lib/python3.12/site-packages (0.33.0)\n",
            "Requirement already satisfied: boto3-stubs>=1.34.110 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from boto3-stubs[bedrock-runtime]>=1.34.110->opik) (1.38.40)\n",
            "Requirement already satisfied: click in /home/mavrick/anaconda3/lib/python3.12/site-packages (from opik) (8.1.8)\n",
            "Requirement already satisfied: httpx in /home/mavrick/anaconda3/lib/python3.12/site-packages (from opik) (0.27.0)\n",
            "Requirement already satisfied: rapidfuzz<4.0.0,>=3.0.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from opik) (3.13.0)\n",
            "Requirement already satisfied: litellm in /home/mavrick/anaconda3/lib/python3.12/site-packages (from opik) (1.72.7)\n",
            "Requirement already satisfied: openai<2.0.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from opik) (1.88.0)\n",
            "Requirement already satisfied: pydantic-settings!=2.9.0,<3.0.0,>=2.0.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from opik) (2.6.1)\n",
            "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from opik) (2.10.3)\n",
            "Requirement already satisfied: pytest in /home/mavrick/anaconda3/lib/python3.12/site-packages (from opik) (8.3.4)\n",
            "Requirement already satisfied: rich in /home/mavrick/anaconda3/lib/python3.12/site-packages (from opik) (13.9.4)\n",
            "Requirement already satisfied: sentry_sdk>=2.0.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from opik) (2.30.0)\n",
            "Requirement already satisfied: tenacity in /home/mavrick/anaconda3/lib/python3.12/site-packages (from opik) (9.0.0)\n",
            "Requirement already satisfied: uuid6 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from opik) (2025.0.0)\n",
            "Requirement already satisfied: jinja2 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from opik) (3.1.6)\n",
            "Requirement already satisfied: filelock in /home/mavrick/anaconda3/lib/python3.12/site-packages (from datasets) (3.17.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from datasets) (1.26.4)\n",
            "Requirement already satisfied: pyarrow>=15.0.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from datasets) (19.0.0)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from datasets) (0.3.8)\n",
            "Requirement already satisfied: requests>=2.32.2 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from datasets) (2.32.3)\n",
            "Requirement already satisfied: xxhash in /home/mavrick/anaconda3/lib/python3.12/site-packages (from datasets) (3.5.0)\n",
            "Requirement already satisfied: multiprocess<0.70.17 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from datasets) (0.70.16)\n",
            "Requirement already satisfied: fsspec<=2025.3.0,>=2023.1.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from fsspec[http]<=2025.3.0,>=2023.1.0->datasets) (2024.12.0)\n",
            "Requirement already satisfied: packaging in /home/mavrick/anaconda3/lib/python3.12/site-packages (from datasets) (24.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from datasets) (6.0.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from transformers) (2024.11.6)\n",
            "Requirement already satisfied: tokenizers<0.22,>=0.21 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from transformers) (0.21.1)\n",
            "Requirement already satisfied: safetensors>=0.4.3 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from transformers) (0.5.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from pandas) (2.9.0.post0)\n",
            "Requirement already satisfied: pytz>=2020.1 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from pandas) (2024.1)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from pandas) (2023.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from huggingface_hub) (4.12.2)\n",
            "Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from huggingface_hub) (1.1.4)\n",
            "Requirement already satisfied: botocore-stubs in /home/mavrick/anaconda3/lib/python3.12/site-packages (from boto3-stubs>=1.34.110->boto3-stubs[bedrock-runtime]>=1.34.110->opik) (1.38.30)\n",
            "Requirement already satisfied: types-s3transfer in /home/mavrick/anaconda3/lib/python3.12/site-packages (from boto3-stubs>=1.34.110->boto3-stubs[bedrock-runtime]>=1.34.110->opik) (0.13.0)\n",
            "Requirement already satisfied: mypy-boto3-bedrock-runtime<1.39.0,>=1.38.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from boto3-stubs[bedrock-runtime]>=1.34.110->opik) (1.38.4)\n",
            "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from fsspec[http]<=2025.3.0,>=2023.1.0->datasets) (3.11.10)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from openai<2.0.0->opik) (4.6.2)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from openai<2.0.0->opik) (1.9.0)\n",
            "Requirement already satisfied: jiter<1,>=0.4.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from openai<2.0.0->opik) (0.10.0)\n",
            "Requirement already satisfied: sniffio in /home/mavrick/anaconda3/lib/python3.12/site-packages (from openai<2.0.0->opik) (1.3.0)\n",
            "Requirement already satisfied: certifi in /home/mavrick/anaconda3/lib/python3.12/site-packages (from httpx->opik) (2025.1.31)\n",
            "Requirement already satisfied: httpcore==1.* in /home/mavrick/anaconda3/lib/python3.12/site-packages (from httpx->opik) (1.0.2)\n",
            "Requirement already satisfied: idna in /home/mavrick/anaconda3/lib/python3.12/site-packages (from httpx->opik) (3.7)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from httpcore==1.*->httpx->opik) (0.14.0)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.0.0->opik) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.27.1 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.0.0->opik) (2.27.1)\n",
            "Requirement already satisfied: python-dotenv>=0.21.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from pydantic-settings!=2.9.0,<3.0.0,>=2.0.0->opik) (0.21.0)\n",
            "Requirement already satisfied: six>=1.5 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from requests>=2.32.2->datasets) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from requests>=2.32.2->datasets) (2.3.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from jinja2->opik) (3.0.2)\n",
            "Requirement already satisfied: importlib-metadata>=6.8.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from litellm->opik) (8.5.0)\n",
            "Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from litellm->opik) (4.23.0)\n",
            "Requirement already satisfied: tiktoken>=0.7.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from litellm->opik) (0.9.0)\n",
            "Requirement already satisfied: iniconfig in /home/mavrick/anaconda3/lib/python3.12/site-packages (from pytest->opik) (1.1.1)\n",
            "Requirement already satisfied: pluggy<2,>=1.5 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from pytest->opik) (1.5.0)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from rich->opik) (2.2.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from rich->opik) (2.15.1)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets) (2.4.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets) (1.2.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets) (24.3.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets) (1.5.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets) (6.1.0)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets) (0.3.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.17.0 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets) (1.18.0)\n",
            "Requirement already satisfied: zipp>=3.20 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from importlib-metadata>=6.8.0->litellm->opik) (3.21.0)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm->opik) (2023.7.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm->opik) (0.30.2)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm->opik) (0.22.3)\n",
            "Requirement already satisfied: mdurl~=0.1 in /home/mavrick/anaconda3/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich->opik) (0.1.0)\n",
            "Requirement already satisfied: types-awscrt in /home/mavrick/anaconda3/lib/python3.12/site-packages (from botocore-stubs->boto3-stubs>=1.34.110->boto3-stubs[bedrock-runtime]>=1.34.110->opik) (0.27.2)\n",
            "Note: you may need to restart the kernel to use updated packages.\n"
          ]
        }
      ],
      "source": [
        "%pip install opik datasets transformers pandas tqdm huggingface_hub --upgrade"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VGYF-bgPqBK4"
      },
      "source": [
        "This step configures the Opik library for your session. It will prompt for your Comet API key if not already set in your environment or through Opik's configuration."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "H0DNm-un_0Np"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "OPIK: Opik is already configured. You can check the settings by viewing the config file at /home/mavrick/.opik.config\n"
          ]
        }
      ],
      "source": [
        "import opik\n",
        "opik.configure()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bdatbf6yqBK4"
      },
      "source": [
        "For this example, we'll primarily use Hugging Face public datasets, but if you need private datasets, you can set your Hugging Face token:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "vN72mHQy_7Ou"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import getpass\n",
        "# Uncomment if you need to access private datasets\n",
        "# if \"HUGGINGFACE_TOKEN\" not in os.environ:\n",
        "#     os.environ[\"HUGGINGFACE_TOKEN\"] = getpass.getpass(\"Enter your Hugging Face token: \")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0dxKlG_0qBK5"
      },
      "source": [
        "## Creating the HuggingFace to Opik Converter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "o2ObsfmBqBK5"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import pandas as pd\n",
        "from datasets import load_dataset, Dataset as HFDataset\n",
        "from opik import Opik\n",
        "from typing import Dict, Any, Optional, List\n",
        "import json\n",
        "from tqdm import tqdm\n",
        "import warnings\n",
        "import numpy as np\n",
        "warnings.filterwarnings('ignore')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "S7iIuIhjqBK5"
      },
      "outputs": [],
      "source": [
        "class HuggingFaceToOpikConverter:\n",
        "    \"\"\"Utility class to convert Hugging Face datasets to Opik format.\"\"\"\n",
        "    \n",
        "    def __init__(self, opik_client: Opik):\n",
        "        self.opik_client = opik_client\n",
        "    \n",
        "    def load_hf_dataset(\n",
        "        self, \n",
        "        dataset_name: str, \n",
        "        split: Optional[str] = None,\n",
        "        config: Optional[str] = None,\n",
        "        subset_size: Optional[int] = None,\n",
        "        **kwargs\n",
        "    ) -> HFDataset:\n",
        "        \"\"\"\n",
        "        Load a dataset from Hugging Face Hub.\n",
        "        \n",
        "        Args:\n",
        "            dataset_name: Name of the dataset on HF Hub\n",
        "            split: Specific split to load (train, validation, test)\n",
        "            config: Configuration/subset of the dataset\n",
        "            subset_size: Limit the number of samples\n",
        "            **kwargs: Additional arguments for load_dataset\n",
        "        \n",
        "        Returns:\n",
        "            Loaded Hugging Face dataset\n",
        "        \"\"\"\n",
        "        print(f\"📥 Loading dataset: {dataset_name}\")\n",
        "        if config:\n",
        "            print(f\"   Config: {config}\")\n",
        "        if split:\n",
        "            print(f\"   Split: {split}\")\n",
        "        \n",
        "        # Load the dataset\n",
        "        dataset = load_dataset(\n",
        "            dataset_name, \n",
        "            name=config,\n",
        "            split=split,\n",
        "            **kwargs\n",
        "        )\n",
        "        \n",
        "        # Limit dataset size if specified\n",
        "        if subset_size and len(dataset) > subset_size:\n",
        "            dataset = dataset.select(range(subset_size))\n",
        "            print(f\"   Limited to {subset_size} samples\")\n",
        "        \n",
        "        print(f\"   ✅ Loaded {len(dataset)} samples\")\n",
        "        print(f\"   Features: {list(dataset.features.keys())}\")\n",
        "        \n",
        "        return dataset\n",
        "    \n",
        "    def _extract_field_value(self, row, column_name):\n",
        "        \"\"\"Extract and convert field value to string if needed.\"\"\"\n",
        "        value = row[column_name]\n",
        "        \n",
        "        # Handle different data types\n",
        "        try:\n",
        "            if isinstance(value, (list, dict)):\n",
        "                # Convert numpy arrays to regular Python objects before JSON serialization\n",
        "                if isinstance(value, dict):\n",
        "                    value = self._convert_numpy_to_python(value)\n",
        "                elif isinstance(value, list):\n",
        "                    value = [self._convert_numpy_to_python(item) for item in value]\n",
        "                return json.dumps(value)\n",
        "            elif isinstance(value, np.ndarray):\n",
        "                # Convert numpy array to list then to JSON\n",
        "                return json.dumps(value.tolist())\n",
        "            elif pd.isna(value):\n",
        "                return \"\"\n",
        "            else:\n",
        "                return str(value)\n",
        "        except (TypeError, ValueError) as e:\n",
        "            # If JSON serialization fails, convert to string representation\n",
        "            print(f\"Warning: Could not serialize value for column '{column_name}': {e}\")\n",
        "            return str(value)\n",
        "    \n",
        "    def _convert_numpy_to_python(self, obj):\n",
        "        \"\"\"Recursively convert numpy objects to Python native types.\"\"\"\n",
        "        if isinstance(obj, np.ndarray):\n",
        "            return obj.tolist()\n",
        "        elif isinstance(obj, np.integer):\n",
        "            return int(obj)\n",
        "        elif isinstance(obj, np.floating):\n",
        "            return float(obj)\n",
        "        elif isinstance(obj, np.bool_):\n",
        "            return bool(obj)\n",
        "        elif isinstance(obj, dict):\n",
        "            return {key: self._convert_numpy_to_python(value) for key, value in obj.items()}\n",
        "        elif isinstance(obj, list):\n",
        "            return [self._convert_numpy_to_python(item) for item in obj]\n",
        "        else:\n",
        "            return obj\n",
        "    \n",
        "    def convert_to_opik_format(\n",
        "        self, \n",
        "        hf_dataset: HFDataset,\n",
        "        input_column: str,\n",
        "        output_column: Optional[str] = None,\n",
        "        expected_output_column: Optional[str] = None,\n",
        "        metadata_columns: Optional[List[str]] = None,\n",
        "        custom_mapping: Optional[Dict[str, str]] = None\n",
        "    ) -> List[Dict[str, Any]]:\n",
        "        \"\"\"\n",
        "        Convert HF dataset to Opik-compatible format.\n",
        "        \n",
        "        Args:\n",
        "            hf_dataset: Hugging Face dataset\n",
        "            input_column: Column to use as input\n",
        "            output_column: Column to use as output (optional)\n",
        "            expected_output_column: Column to use as expected output (optional)\n",
        "            metadata_columns: Additional columns to include as metadata\n",
        "            custom_mapping: Custom mapping of HF columns to Opik fields\n",
        "        \n",
        "        Returns:\n",
        "            List of Opik dataset items\n",
        "        \"\"\"\n",
        "        opik_items = []\n",
        "        metadata_columns = metadata_columns or []\n",
        "        \n",
        "        # Convert to pandas for easier manipulation\n",
        "        df = hf_dataset.to_pandas()\n",
        "        \n",
        "        print(f\"🔄 Converting {len(df)} samples to Opik format...\")\n",
        "        \n",
        "        for idx, row in tqdm(df.iterrows(), total=len(df), desc=\"Converting\"):\n",
        "            # Build Opik item\n",
        "            opik_item = {\n",
        "                \"input\": self._extract_field_value(row, input_column)\n",
        "            }\n",
        "            \n",
        "            # Add output if specified\n",
        "            if output_column and output_column in df.columns:\n",
        "                opik_item[\"output\"] = self._extract_field_value(row, output_column)\n",
        "            \n",
        "            # Add expected output if specified\n",
        "            if expected_output_column and expected_output_column in df.columns:\n",
        "                opik_item[\"expected_output\"] = self._extract_field_value(row, expected_output_column)\n",
        "            \n",
        "            # Add metadata columns\n",
        "            if metadata_columns:\n",
        "                metadata = {}\n",
        "                for col in metadata_columns:\n",
        "                    if col in df.columns:\n",
        "                        metadata[col] = self._extract_field_value(row, col)\n",
        "                if metadata:\n",
        "                    opik_item[\"metadata\"] = metadata\n",
        "            \n",
        "            # Apply custom mapping\n",
        "            if custom_mapping:\n",
        "                for hf_col, opik_field in custom_mapping.items():\n",
        "                    if hf_col in df.columns:\n",
        "                        opik_item[opik_field] = self._extract_field_value(row, hf_col)\n",
        "            \n",
        "            opik_items.append(opik_item)\n",
        "        \n",
        "        print(f\"   ✅ Converted {len(opik_items)} items\")\n",
        "        return opik_items\n",
        "    \n",
        "    def create_opik_dataset(\n",
        "        self,\n",
        "        dataset_name: str,\n",
        "        opik_items: List[Dict[str, Any]],\n",
        "        description: Optional[str] = None\n",
        "    ):\n",
        "        \"\"\"Create or update an Opik dataset with the converted items.\"\"\"\n",
        "        print(f\"📤 Creating Opik dataset: {dataset_name}\")\n",
        "        \n",
        "        # Create or get existing dataset\n",
        "        dataset = self.opik_client.get_or_create_dataset(\n",
        "            name=dataset_name,\n",
        "            description=description or f\"Dataset imported from Hugging Face\"\n",
        "        )\n",
        "        \n",
        "        # Insert items in batches\n",
        "        batch_size = 100\n",
        "        \n",
        "        for i in tqdm(range(0, len(opik_items), batch_size), desc=\"Uploading batches\"):\n",
        "            batch = opik_items[i:i + batch_size]\n",
        "            dataset.insert(batch)\n",
        "        \n",
        "        print(f\"   ✅ Successfully created dataset with {len(opik_items)} items\")\n",
        "        return dataset\n",
        "    \n",
        "    def import_dataset(\n",
        "        self,\n",
        "        hf_dataset_name: str,\n",
        "        opik_dataset_name: str,\n",
        "        input_column: str,\n",
        "        split: Optional[str] = \"train\",\n",
        "        config: Optional[str] = None,\n",
        "        output_column: Optional[str] = None,\n",
        "        expected_output_column: Optional[str] = None,\n",
        "        metadata_columns: Optional[List[str]] = None,\n",
        "        custom_mapping: Optional[Dict[str, str]] = None,\n",
        "        subset_size: Optional[int] = None,\n",
        "        description: Optional[str] = None\n",
        "    ):\n",
        "        \"\"\"Complete pipeline to import HF dataset to Opik.\"\"\"\n",
        "        print(f\"🚀 Starting import: {hf_dataset_name} → {opik_dataset_name}\")\n",
        "        \n",
        "        # Load HF dataset\n",
        "        hf_dataset = self.load_hf_dataset(\n",
        "            hf_dataset_name, \n",
        "            split=split, \n",
        "            config=config, \n",
        "            subset_size=subset_size\n",
        "        )\n",
        "        \n",
        "        # Convert to Opik format\n",
        "        opik_items = self.convert_to_opik_format(\n",
        "            hf_dataset,\n",
        "            input_column=input_column,\n",
        "            output_column=output_column,\n",
        "            expected_output_column=expected_output_column,\n",
        "            metadata_columns=metadata_columns,\n",
        "            custom_mapping=custom_mapping\n",
        "        )\n",
        "        \n",
        "        # Create Opik dataset\n",
        "        dataset = self.create_opik_dataset(\n",
        "            opik_dataset_name,\n",
        "            opik_items,\n",
        "            description=description\n",
        "        )\n",
        "        \n",
        "        print(f\"🎉 Import completed successfully!\")\n",
        "        return dataset\n",
        "\n",
        "# Initialize the converter\n",
        "opik_client = Opik()\n",
        "converter = HuggingFaceToOpikConverter(opik_client)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M7QWFwwZqBK5"
      },
      "source": [
        "## Example 1: Importing SQuAD Dataset\n",
        "\n",
        "Let's start with a popular question-answering dataset:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "7Ud1WzmVqBK5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "📚 Example 1: Importing SQuAD Dataset\n",
            "==================================================\n",
            "🚀 Starting import: rajpurkar/squad → squad-qa-dataset\n",
            "📥 Loading dataset: rajpurkar/squad\n",
            "   Split: train\n",
            "   Limited to 1000 samples\n",
            "   ✅ Loaded 1000 samples\n",
            "   Features: ['id', 'title', 'context', 'question', 'answers']\n",
            "🔄 Converting 1000 samples to Opik format...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Converting: 100%|██████████| 1000/1000 [00:00<00:00, 38154.66it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "   ✅ Converted 1000 items\n",
            "📤 Creating Opik dataset: squad-qa-dataset\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "OPIK: Created a \"squad-qa-dataset\" dataset at https://www.comet.com/opik/api/v1/session/redirect/datasets/?dataset_id=01978ee2-efe6-7a52-8d0a-698e2c41ac65&path=aHR0cHM6Ly93d3cuY29tZXQuY29tL29waWsvYXBpLw==.\n",
            "Uploading batches: 100%|██████████| 10/10 [00:04<00:00,  2.47it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "   ✅ Successfully created dataset with 1000 items\n",
            "🎉 Import completed successfully!\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "print(\"📚 Example 1: Importing SQuAD Dataset\")\n",
        "print(\"=\" * 50)\n",
        "\n",
        "squad_dataset = converter.import_dataset(\n",
        "    hf_dataset_name=\"rajpurkar/squad\",\n",
        "    opik_dataset_name=\"squad-qa-dataset\",\n",
        "    input_column=\"question\",\n",
        "    expected_output_column=\"answers\",\n",
        "    split=\"train\",\n",
        "    subset_size=1000,  # Limit to 1000 samples for demo\n",
        "    metadata_columns=[\"context\", \"id\"],\n",
        "    description=\"SQuAD question-answering dataset imported from Hugging Face\"\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "uI3_Me7LqBK6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Found 1000 items in SQuAD dataset\n"
          ]
        }
      ],
      "source": [
        "print(f\"Found {len(squad_dataset.to_pandas())} items in SQuAD dataset\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NUsTWUV7qBK6"
      },
      "source": [
        "## Example 2: Importing IMDB Movie Reviews\n",
        "\n",
        "Now let's import a sentiment analysis dataset:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "Rs-n7ARuqBK6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "🎬 Example 2: Importing IMDB Movie Reviews\n",
            "==================================================\n",
            "🚀 Starting import: stanfordnlp/imdb → imdb-sentiment-dataset\n",
            "📥 Loading dataset: stanfordnlp/imdb\n",
            "   Split: train\n",
            "   Limited to 500 samples\n",
            "   ✅ Loaded 500 samples\n",
            "   Features: ['text', 'label']\n",
            "🔄 Converting 500 samples to Opik format...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Converting: 100%|██████████| 500/500 [00:00<00:00, 47699.40it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "   ✅ Converted 500 items\n",
            "📤 Creating Opik dataset: imdb-sentiment-dataset\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "OPIK: Created a \"imdb-sentiment-dataset\" dataset at https://www.comet.com/opik/api/v1/session/redirect/datasets/?dataset_id=01978ee3-242d-79f4-82b5-f85923041c1c&path=aHR0cHM6Ly93d3cuY29tZXQuY29tL29waWsvYXBpLw==.\n",
            "Uploading batches: 100%|██████████| 5/5 [00:02<00:00,  1.80it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "   ✅ Successfully created dataset with 500 items\n",
            "🎉 Import completed successfully!\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "print(\"\\n🎬 Example 2: Importing IMDB Movie Reviews\")\n",
        "print(\"=\" * 50)\n",
        "\n",
        "imdb_dataset = converter.import_dataset(\n",
        "    hf_dataset_name=\"stanfordnlp/imdb\",\n",
        "    opik_dataset_name=\"imdb-sentiment-dataset\",\n",
        "    input_column=\"text\",\n",
        "    expected_output_column=\"label\",\n",
        "    split=\"train\",\n",
        "    subset_size=500,  # Limit to 500 samples for demo\n",
        "    description=\"IMDB movie reviews sentiment dataset imported from Hugging Face\"\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M__d8GmXqBK6"
      },
      "source": [
        "## Example 3: Importing GSM8K Math Problems\n",
        "\n",
        "Let's import a mathematical reasoning dataset:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "JLCq9UnBqBK6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "🔢 Example 3: GSM8K Mathematical Reasoning Dataset\n",
            "==================================================\n",
            "🚀 Starting import: openai/gsm8k → gsm8k-math-problems\n",
            "📥 Loading dataset: openai/gsm8k\n",
            "   Config: main\n",
            "   Split: train\n",
            "   Limited to 200 samples\n",
            "   ✅ Loaded 200 samples\n",
            "   Features: ['question', 'answer']\n",
            "🔄 Converting 200 samples to Opik format...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Converting: 100%|██████████| 200/200 [00:00<00:00, 63632.01it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "   ✅ Converted 200 items\n",
            "📤 Creating Opik dataset: gsm8k-math-problems\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "OPIK: Created a \"gsm8k-math-problems\" dataset at https://www.comet.com/opik/api/v1/session/redirect/datasets/?dataset_id=01978ee3-485a-71d4-a377-79b1bd2acfea&path=aHR0cHM6Ly93d3cuY29tZXQuY29tL29waWsvYXBpLw==.\n",
            "Uploading batches: 100%|██████████| 2/2 [00:01<00:00,  1.92it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "   ✅ Successfully created dataset with 200 items\n",
            "🎉 Import completed successfully!\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "print(\"\\n🔢 Example 3: GSM8K Mathematical Reasoning Dataset\")\n",
        "print(\"=\" * 50)\n",
        "\n",
        "gsm8k_dataset = converter.import_dataset(\n",
        "    hf_dataset_name=\"openai/gsm8k\",\n",
        "    opik_dataset_name=\"gsm8k-math-problems\",\n",
        "    config=\"main\",\n",
        "    input_column=\"question\",\n",
        "    expected_output_column=\"answer\",\n",
        "    split=\"train\",\n",
        "    subset_size=200,\n",
        "    description=\"GSM8K mathematical reasoning dataset imported from Hugging Face\"\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8BwrQlpqqBK6"
      },
      "source": [
        "## Example 4: Importing TruthfulQA Dataset\n",
        "\n",
        "Let's import a dataset for evaluating truthfulness:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "ak1FJasAqBK6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "🔍 Example 4: TruthfulQA Dataset\n",
            "==================================================\n",
            "🚀 Starting import: truthful_qa → truthfulqa-dataset\n",
            "📥 Loading dataset: truthful_qa\n",
            "   Config: generation\n",
            "   Split: validation\n",
            "   Limited to 100 samples\n",
            "   ✅ Loaded 100 samples\n",
            "   Features: ['type', 'category', 'question', 'best_answer', 'correct_answers', 'incorrect_answers', 'source']\n",
            "🔄 Converting 100 samples to Opik format...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Converting: 100%|██████████| 100/100 [00:00<00:00, 36076.93it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "   ✅ Converted 100 items\n",
            "📤 Creating Opik dataset: truthfulqa-dataset\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "OPIK: Created a \"truthfulqa-dataset\" dataset at https://www.comet.com/opik/api/v1/session/redirect/datasets/?dataset_id=01978ee3-715e-7f76-8830-e595aa134f3a&path=aHR0cHM6Ly93d3cuY29tZXQuY29tL29waWsvYXBpLw==.\n",
            "Uploading batches: 100%|██████████| 1/1 [00:00<00:00,  2.66it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "   ✅ Successfully created dataset with 100 items\n",
            "🎉 Import completed successfully!\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "print(\"\\n🔍 Example 4: TruthfulQA Dataset\")\n",
        "print(\"=\" * 50)\n",
        "\n",
        "try:\n",
        "    truthfulqa_dataset = converter.import_dataset(\n",
        "        hf_dataset_name=\"truthful_qa\",\n",
        "        opik_dataset_name=\"truthfulqa-dataset\",\n",
        "        config=\"generation\",\n",
        "        input_column=\"question\",\n",
        "        expected_output_column=\"best_answer\",\n",
        "        split=\"validation\",\n",
        "        subset_size=100,\n",
        "        metadata_columns=[\"category\", \"source\"],\n",
        "        description=\"TruthfulQA dataset for evaluating truthfulness imported from Hugging Face\"\n",
        "    )\n",
        "except Exception as e:\n",
        "    print(f\"Note: TruthfulQA import failed: {e}\")\n",
        "    print(\"This might be due to dataset structure changes or access restrictions.\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IWylo1mmqBK6"
      },
      "source": [
        "## Verifying Your Imported Datasets\n",
        "\n",
        "Let's check what datasets we've successfully imported:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "InzrflUwqBK6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "✅ Verification: Listing All Datasets\n",
            "==================================================\n",
            "Total datasets in Opik: 4\n",
            "  📁 truthfulqa-dataset\n",
            "     Description: TruthfulQA dataset for evaluating truthfulness imported from Hugging Face\n",
            "  📁 gsm8k-math-problems\n",
            "     Description: GSM8K mathematical reasoning dataset imported from Hugging Face\n",
            "  📁 imdb-sentiment-dataset\n",
            "     Description: IMDB movie reviews sentiment dataset imported from Hugging Face\n",
            "  📁 squad-qa-dataset\n",
            "     Description: SQuAD question-answering dataset imported from Hugging Face\n"
          ]
        }
      ],
      "source": [
        "print(\"\\n✅ Verification: Listing All Datasets\")\n",
        "print(\"=\" * 50)\n",
        "\n",
        "# List all datasets in Opik\n",
        "datasets = opik_client.get_datasets()\n",
        "print(f\"Total datasets in Opik: {len(datasets)}\")\n",
        "\n",
        "for dataset in datasets:\n",
        "    print(f\"  📁 {dataset.name}\")\n",
        "    if hasattr(dataset, 'description') and dataset.description:\n",
        "        print(f\"     Description: {dataset.description}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M__d8GmXqBK6"
      },
      "source": [
        "You can now go to the Opik app to see your imported datasets:\n",
        "\n",
        "![Hugging Face datasets in Opik](https://raw.githubusercontent.com/comet-ml/opik/main/apps/opik-documentation/documentation/fern/img/cookbook/hugging_face_datasets_in_opik.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N_UxLOuuqBK6"
      },
      "source": [
        "## Using Imported Datasets for Evaluation\n",
        "\n",
        "Here's how you can use the imported datasets for LLM evaluation:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "VBoXzn4UqBK6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "🧪 Example: Using Imported Dataset for Evaluation\n",
            "==================================================\n",
            "Running sample evaluation...\n",
            "\n",
            "Sample 1:\n",
            "Input: What did Beyonce and Rowland found in 2005?...\n",
            "Expected: {\"text\": [\"the Survivor Foundation\"], \"answer_start\": [61]}...\n",
            "Predicted: This is a dummy answer to: What did Beyonce and Rowland found in 2005?...\n",
            "----------------------------------------\n",
            "\n",
            "Sample 2:\n",
            "Input: What hurricane years later after Katrina did the organization provide support for?...\n",
            "Expected: {\"text\": [\"Ike\"], \"answer_start\": [321]}...\n",
            "Predicted: This is a dummy answer to: What hurricane years later after Katrina did the o...\n",
            "----------------------------------------\n",
            "\n",
            "Sample 3:\n",
            "Input: How much cash did Beyonce put into the venture, the Survivor Foundation at startup?...\n",
            "Expected: {\"text\": [\"250,000\"], \"answer_start\": [191]}...\n",
            "Predicted: This is a dummy answer to: How much cash did Beyonce put into the venture, th...\n",
            "----------------------------------------\n"
          ]
        }
      ],
      "source": [
        "print(\"\\n🧪 Example: Using Imported Dataset for Evaluation\")\n",
        "print(\"=\" * 50)\n",
        "\n",
        "from opik.evaluation.metrics import LevenshteinRatio\n",
        "\n",
        "# Example evaluation function\n",
        "def dummy_llm_function(input_question: str) -> str:\n",
        "    \"\"\"\n",
        "    Dummy LLM function for demonstration.\n",
        "    In real use, this would call your actual LLM.\n",
        "    \"\"\"\n",
        "    return f\"This is a dummy answer to: {input_question[:50]}...\"\n",
        "\n",
        "# Get one of our imported datasets\n",
        "try:\n",
        "    squad_dataset = opik_client.get_dataset(\"squad-qa-dataset\")\n",
        "    \n",
        "    # Convert to pandas for easier handling\n",
        "    df = squad_dataset.to_pandas().head(3)  # Just show first 3 for demo\n",
        "    \n",
        "    print(\"Running sample evaluation...\")\n",
        "    for idx, row in df.iterrows():\n",
        "        input_text = row['input']\n",
        "        expected = row.get('expected_output', 'No expected output')\n",
        "        predicted = dummy_llm_function(input_text)\n",
        "        \n",
        "        print(f\"\\nSample {idx + 1}:\")\n",
        "        print(f\"Input: {input_text[:100]}...\")\n",
        "        print(f\"Expected: {str(expected)[:100]}...\")\n",
        "        print(f\"Predicted: {predicted}\")\n",
        "        print(\"-\" * 40)\n",
        "\n",
        "except Exception as e:\n",
        "    print(f\"Evaluation example failed: {e}\")\n",
        "    print(\"Make sure you have successfully imported a dataset first.\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bLYw8z2ZqBK7"
      },
      "source": [
        "## Helper Function for Common Datasets\n",
        "\n",
        "Here's a helper function to quickly import popular evaluation datasets:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "1irsA0AhqBK7"
      },
      "outputs": [],
      "source": [
        "def import_common_datasets(converter, subset_size=200):\n",
        "    \"\"\"Import several popular datasets for LLM evaluation.\"\"\"\n",
        "    \n",
        "    common_datasets = [\n",
        "        {\n",
        "            \"hf_name\": \"ai2_arc\",\n",
        "            \"opik_name\": \"arc-reasoning-dataset\", \n",
        "            \"config\": \"ARC-Challenge\",\n",
        "            \"input_col\": \"question\",\n",
        "            \"expected_col\": \"answerKey\",\n",
        "            \"metadata_cols\": [\"choices\"],\n",
        "            \"description\": \"AI2 ARC reasoning dataset\"\n",
        "        },\n",
        "        {\n",
        "            \"hf_name\": \"winogrande\",\n",
        "            \"opik_name\": \"winogrande-commonsense-dataset\",\n",
        "            \"config\": \"winogrande_debiased\",\n",
        "            \"input_col\": \"sentence\",\n",
        "            \"expected_col\": \"answer\",\n",
        "            \"metadata_cols\": [\"option1\", \"option2\"],\n",
        "            \"description\": \"WinoGrande commonsense reasoning dataset\"\n",
        "        },\n",
        "        {\n",
        "            \"hf_name\": \"hellaswag\",\n",
        "            \"opik_name\": \"hellaswag-dataset\",\n",
        "            \"input_col\": \"ctx\",\n",
        "            \"expected_col\": \"label\",\n",
        "            \"metadata_cols\": [\"endings\", \"source_id\"],\n",
        "            \"description\": \"HellaSwag commonsense reasoning dataset\"\n",
        "        }\n",
        "    ]\n",
        "    \n",
        "    for dataset_config in common_datasets:\n",
        "        try:\n",
        "            print(f\"\\n📥 Importing {dataset_config['hf_name']}...\")\n",
        "            converter.import_dataset(\n",
        "                hf_dataset_name=dataset_config[\"hf_name\"],\n",
        "                opik_dataset_name=dataset_config[\"opik_name\"],\n",
        "                config=dataset_config.get(\"config\"),\n",
        "                input_column=dataset_config[\"input_col\"],\n",
        "                expected_output_column=dataset_config[\"expected_col\"],\n",
        "                metadata_columns=dataset_config.get(\"metadata_cols\"),\n",
        "                subset_size=subset_size,\n",
        "                description=dataset_config[\"description\"],\n",
        "                split=\"validation\"  # Use validation split for evaluation\n",
        "            )\n",
        "        except Exception as e:\n",
        "            print(f\"❌ Failed to import {dataset_config['hf_name']}: {str(e)}\")\n",
        "            print(\"   This might be due to dataset structure changes or access restrictions.\")\n",
        "\n",
        "# Uncomment to import common datasets\n",
        "# import_common_datasets(converter, subset_size=100)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G2CUNgLBqBK7"
      },
      "source": [
        "## Advanced Usage: Custom Dataset Processing\n",
        "\n",
        "For more complex datasets, you might need custom processing:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "8whG4mHVqBK7"
      },
      "outputs": [],
      "source": [
        "def import_custom_translation_dataset():\n",
        "    \"\"\"Example of importing a translation dataset with custom processing.\"\"\"\n",
        "    \n",
        "    print(\"\\n🌐 Advanced Example: Translation Dataset\")\n",
        "    print(\"=\" * 50)\n",
        "    \n",
        "    try:\n",
        "        # Load a translation dataset\n",
        "        from datasets import load_dataset\n",
        "        \n",
        "        # Example with a simple translation dataset\n",
        "        dataset = load_dataset(\"Helsinki-NLP/opus-100\", \"en-es\", split=\"train\")\n",
        "        \n",
        "        # Custom processing for translation pairs\n",
        "        opik_items = []\n",
        "        for i, item in enumerate(dataset.select(range(100))):  # Limit for demo\n",
        "            # Extract source and target from translation object\n",
        "            translation = item['translation']\n",
        "            opik_item = {\n",
        "                \"input\": translation['en'],  # English source\n",
        "                \"expected_output\": translation['es'],  # Spanish target\n",
        "                \"metadata\": {\n",
        "                    \"language_pair\": \"en-es\",\n",
        "                    \"domain\": \"general\",\n",
        "                    \"sample_id\": i\n",
        "                }\n",
        "            }\n",
        "            opik_items.append(opik_item)\n",
        "        \n",
        "        # Create Opik dataset\n",
        "        dataset = opik_client.get_or_create_dataset(\n",
        "            name=\"opus-translation-en-es\",\n",
        "            description=\"OPUS-100 translation dataset (English to Spanish)\"\n",
        "        )\n",
        "        \n",
        "        # Insert items in batches\n",
        "        batch_size = 50\n",
        "        for i in tqdm(range(0, len(opik_items), batch_size), desc=\"Uploading\"):\n",
        "            batch = opik_items[i:i + batch_size]\n",
        "            dataset.insert(batch)\n",
        "            \n",
        "        print(f\"✅ Successfully imported {len(opik_items)} translation pairs\")\n",
        "        \n",
        "    except Exception as e:\n",
        "        print(f\"Translation dataset example failed: {e}\")\n",
        "        print(\"This is expected as dataset structures can vary.\")\n",
        "\n",
        "# Uncomment to try the translation example\n",
        "# import_custom_translation_dataset()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9eD9lYxAqBK7"
      },
      "source": [
        "## Store New Dataset in Opik\n",
        "\n",
        "This section shows how to create a dataset programmatically similar to the synthetic data generation example:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "8C8_TbxMqBK9"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "OPIK: Created a \"sample-qa-dataset\" dataset at https://www.comet.com/opik/api/v1/session/redirect/datasets/?dataset_id=01978ee3-9176-74b9-be43-472ff2adb897&path=aHR0cHM6Ly93d3cuY29tZXQuY29tL29waWsvYXBpLw==.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Opik Dataset 'sample-qa-dataset' created with ID: 01978ee3-9176-74b9-be43-472ff2adb897\n"
          ]
        }
      ],
      "source": [
        "def create_sample_qa_dataset():\n",
        "    \"\"\"Create a sample Q&A dataset similar to the synthetic data optimizer example.\"\"\"\n",
        "    \n",
        "    sample_items = [\n",
        "        {\n",
        "            \"question\": \"What is the capital of France?\",\n",
        "            \"answer\": \"Paris\",\n",
        "            \"category\": \"geography\",\n",
        "            \"difficulty\": \"easy\"\n",
        "        },\n",
        "        {\n",
        "            \"question\": \"Who wrote Romeo and Juliet?\",\n",
        "            \"answer\": \"William Shakespeare\", \n",
        "            \"category\": \"literature\",\n",
        "            \"difficulty\": \"easy\"\n",
        "        },\n",
        "        {\n",
        "            \"question\": \"What is the largest planet in our solar system?\",\n",
        "            \"answer\": \"Jupiter\",\n",
        "            \"category\": \"science\",\n",
        "            \"difficulty\": \"medium\"\n",
        "        },\n",
        "        {\n",
        "            \"question\": \"In what year did World War II end?\",\n",
        "            \"answer\": \"1945\",\n",
        "            \"category\": \"history\", \n",
        "            \"difficulty\": \"medium\"\n",
        "        },\n",
        "        {\n",
        "            \"question\": \"What is the chemical symbol for gold?\",\n",
        "            \"answer\": \"Au\",\n",
        "            \"category\": \"science\",\n",
        "            \"difficulty\": \"hard\"\n",
        "        }\n",
        "    ]\n",
        "    \n",
        "    dataset_name = \"sample-qa-dataset\"\n",
        "    \n",
        "    dataset = opik_client.get_or_create_dataset(\n",
        "        name=dataset_name,\n",
        "        description=\"Sample Q&A dataset for demonstration\"\n",
        "    )\n",
        "    \n",
        "    dataset.insert(sample_items)\n",
        "    print(f\"Opik Dataset '{dataset.name}' created with ID: {dataset.id}\")\n",
        "    \n",
        "    return dataset\n",
        "\n",
        "# Create the sample dataset\n",
        "sample_dataset = create_sample_qa_dataset()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "![sample-qa-dataset](https://raw.githubusercontent.com/comet-ml/opik/main/apps/opik-documentation/documentation/fern/img/cookbook/hugging_face_datasets_in_opik_sample-qa-dataset.png)\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "base",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.7"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}