{ "cells": [ { "cell_type": "markdown", "id": "876eb80c", "metadata": { "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5" }, "source": [ "# Describe product images with BigFrames multimodal DataFrames\n", "\n", "Based on notebook at https://github.com/googleapis/python-bigquery-dataframes/blob/main/notebooks/multimodal/multimodal_dataframe.ipynb\n", "\n", "This notebook is introducing BigFrames Multimodal features:\n", "\n", "1. Create Multimodal DataFrame\n", "2. Combine unstructured data with structured data\n", "3. Conduct image transformations\n", "4. Use LLM models to ask questions and generate embeddings on images\n", "5. PDF chunking function\n", "\n", "Install the bigframes package and upgrade other packages that are already included in Kaggle but have versions incompatible with bigframes." ] }, { "cell_type": "code", "execution_count": 1, "id": "0506e15e", "metadata": { "trusted": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: bigframes in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (2.39.0)\n", "Requirement already satisfied: google-cloud-automl in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (2.19.0)\n", "Requirement already satisfied: google-cloud-translate in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (3.26.0)\n", "Requirement already satisfied: google-ai-generativelanguage in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (0.11.0)\n", "Requirement already satisfied: tensorflow in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (2.21.0)\n", "Requirement already satisfied: cloudpickle>=2.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (3.1.2)\n", "Requirement already satisfied: fsspec>=2023.3.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2026.1.0)\n", "Requirement already satisfied: gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2026.1.0)\n", "Requirement already satisfied: geopandas>=0.12.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.1.3)\n", "Requirement already satisfied: google-auth<3.0,>=2.15.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.49.1)\n", "Requirement already satisfied: google-cloud-bigquery>=3.36.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-cloud-bigquery[bqstorage,pandas]>=3.36.0->bigframes) (3.41.0)\n", "Requirement already satisfied: google-cloud-bigquery-storage<3.0.0,>=2.30.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.37.0)\n", "Requirement already satisfied: google-cloud-functions>=1.12.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.23.0)\n", "Requirement already satisfied: google-cloud-bigquery-connection>=1.12.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.21.0)\n", "Requirement already satisfied: google-cloud-resource-manager>=1.10.3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.17.0)\n", "Requirement already satisfied: google-cloud-storage>=2.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (3.10.1)\n", "Requirement already satisfied: google-crc32c<2.0.0,>=1.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.8.0)\n", "Requirement already satisfied: grpc-google-iam-v1>=0.14.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (0.14.4)\n", "Requirement already satisfied: numpy>=1.24.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.4.4)\n", "Requirement already satisfied: pandas>=1.5.3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.3.3)\n", "Requirement already satisfied: pandas-gbq>=0.26.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (0.34.1)\n", "Requirement already satisfied: pyarrow>=15.0.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (21.0.0)\n", "Requirement already satisfied: pydata-google-auth>=1.8.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.9.1)\n", "Requirement already satisfied: requests>=2.27.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.33.1)\n", "Requirement already satisfied: shapely>=1.8.5 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.1.2)\n", "Requirement already satisfied: tabulate>=0.9 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (0.10.0)\n", "Requirement already satisfied: humanize>=4.6.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (4.15.0)\n", "Requirement already satisfied: matplotlib>=3.7.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (3.10.8)\n", "Requirement already satisfied: db-dtypes>=1.4.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.5.1)\n", "Requirement already satisfied: pyiceberg>=0.7.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (0.11.1)\n", "Requirement already satisfied: atpublic<6,>=2.3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (5.1)\n", "Requirement already satisfied: python-dateutil<3,>=2.8.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2.9.0.post0)\n", "Requirement already satisfied: pytz>=2022.7 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (2026.1.post1)\n", "Requirement already satisfied: toolz<2,>=0.11 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (1.1.0)\n", "Requirement already satisfied: typing-extensions<5,>=4.5.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (4.15.0)\n", "Requirement already satisfied: rich<14,>=12.4.4 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from bigframes) (13.9.4)\n", "Requirement already satisfied: google-api-core<3.0.0,>=2.11.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-api-core[grpc]<3.0.0,>=2.11.0->google-cloud-automl) (2.30.2)\n", "Requirement already satisfied: grpcio<2.0.0,>=1.33.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-cloud-automl) (1.80.0)\n", "Requirement already satisfied: proto-plus<2.0.0,>=1.22.3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-cloud-automl) (1.27.2)\n", "Requirement already satisfied: protobuf<8.0.0,>=4.25.8 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-cloud-automl) (6.33.6)\n", "Requirement already satisfied: google-cloud-core<3.0.0,>=2.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-cloud-translate) (2.5.1)\n", "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (2.4.0)\n", "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (1.6.3)\n", "Requirement already satisfied: flatbuffers>=25.9.23 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (25.12.19)\n", "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (0.7.0)\n", "Requirement already satisfied: google_pasta>=0.1.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (0.2.0)\n", "Requirement already satisfied: libclang>=13.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (18.1.1)\n", "Requirement already satisfied: opt_einsum>=2.3.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (3.4.0)\n", "Requirement already satisfied: packaging in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (26.0)\n", "Requirement already satisfied: setuptools in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (82.0.1)\n", "Requirement already satisfied: six>=1.12.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (1.17.0)\n", "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (3.3.0)\n", "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (2.1.2)\n", "Requirement already satisfied: keras>=3.12.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (3.14.0)\n", "Requirement already satisfied: h5py<3.15.0,>=3.11.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (3.14.0)\n", "Requirement already satisfied: ml_dtypes<1.0.0,>=0.5.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from tensorflow) (0.5.4)\n", "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from astunparse>=1.6.0->tensorflow) (0.47.0)\n", "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (3.13.5)\n", "Requirement already satisfied: decorator>4.1.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (5.2.1)\n", "Requirement already satisfied: google-auth-oauthlib in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (1.3.1)\n", "Requirement already satisfied: google-cloud-storage-control in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (1.11.0)\n", "Requirement already satisfied: pyogrio>=0.7.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from geopandas>=0.12.2->bigframes) (0.12.1)\n", "Requirement already satisfied: pyproj>=3.5.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from geopandas>=0.12.2->bigframes) (3.7.2)\n", "Requirement already satisfied: googleapis-common-protos<2.0.0,>=1.63.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-api-core<3.0.0,>=2.11.0->google-api-core[grpc]<3.0.0,>=2.11.0->google-cloud-automl) (1.74.0)\n", "Requirement already satisfied: grpcio-status<2.0.0,>=1.33.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-api-core[grpc]<3.0.0,>=2.11.0->google-cloud-automl) (1.80.0)\n", "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-auth<3.0,>=2.15.0->bigframes) (0.4.2)\n", "Requirement already satisfied: cryptography>=38.0.3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-auth<3.0,>=2.15.0->bigframes) (46.0.7)\n", "Requirement already satisfied: google-resumable-media<3.0.0,>=2.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-cloud-bigquery>=3.36.0->google-cloud-bigquery[bqstorage,pandas]>=3.36.0->bigframes) (2.8.2)\n", "Requirement already satisfied: namex in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from keras>=3.12.0->tensorflow) (0.1.0)\n", "Requirement already satisfied: optree in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from keras>=3.12.0->tensorflow) (0.19.0)\n", "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from matplotlib>=3.7.1->bigframes) (1.3.3)\n", "Requirement already satisfied: cycler>=0.10 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from matplotlib>=3.7.1->bigframes) (0.12.1)\n", "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from matplotlib>=3.7.1->bigframes) (4.62.1)\n", "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from matplotlib>=3.7.1->bigframes) (1.5.0)\n", "Requirement already satisfied: pillow>=8 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from matplotlib>=3.7.1->bigframes) (12.2.0)\n", "Requirement already satisfied: pyparsing>=3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from matplotlib>=3.7.1->bigframes) (3.3.2)\n", "Requirement already satisfied: tzdata>=2022.7 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pandas>=1.5.3->bigframes) (2026.1)\n", "Requirement already satisfied: psutil>=5.9.8 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pandas-gbq>=0.26.1->bigframes) (7.2.2)\n", "Requirement already satisfied: mmh3<6.0.0,>=4.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (5.2.1)\n", "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (8.3.2)\n", "Requirement already satisfied: strictyaml<2.0.0,>=1.7.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (1.7.3)\n", "Requirement already satisfied: pydantic!=2.12.0,!=2.12.1,!=2.4.0,!=2.4.1,<3.0,>=2.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (2.12.5)\n", "Requirement already satisfied: tenacity<10.0.0,>=8.2.3 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (9.1.4)\n", "Requirement already satisfied: pyroaring<2.0.0,>=1.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (1.0.4)\n", "Requirement already satisfied: cachetools<7.0,>=5.5 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (6.2.6)\n", "Requirement already satisfied: zstandard<1.0.0,>=0.13.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyiceberg>=0.7.1->bigframes) (0.25.0)\n", "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from requests>=2.27.1->bigframes) (3.4.7)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from requests>=2.27.1->bigframes) (3.11)\n", "Requirement already satisfied: urllib3<3,>=1.26 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from requests>=2.27.1->bigframes) (2.6.3)\n", "Requirement already satisfied: certifi>=2023.5.7 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from requests>=2.27.1->bigframes) (2026.2.25)\n", "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from rich<14,>=12.4.4->bigframes) (4.0.0)\n", "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from rich<14,>=12.4.4->bigframes) (2.20.0)\n", "Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (2.6.1)\n", "Requirement already satisfied: aiosignal>=1.4.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (1.4.0)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (26.1.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (1.8.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (6.7.1)\n", "Requirement already satisfied: propcache>=0.2.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (0.4.1)\n", "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (1.23.0)\n", "Requirement already satisfied: cffi>=2.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from cryptography>=38.0.3->google-auth<3.0,>=2.15.0->bigframes) (2.0.0)\n", "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from google-auth-oauthlib->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (2.0.0)\n", "Requirement already satisfied: mdurl~=0.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from markdown-it-py>=2.2.0->rich<14,>=12.4.4->bigframes) (0.1.2)\n", "Requirement already satisfied: pyasn1<0.7.0,>=0.6.1 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0,>=2.15.0->bigframes) (0.6.3)\n", "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pydantic!=2.12.0,!=2.12.1,!=2.4.0,!=2.4.1,<3.0,>=2.0->pyiceberg>=0.7.1->bigframes) (0.7.0)\n", "Requirement already satisfied: pydantic-core==2.41.5 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pydantic!=2.12.0,!=2.12.1,!=2.4.0,!=2.4.1,<3.0,>=2.0->pyiceberg>=0.7.1->bigframes) (2.41.5)\n", "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from pydantic!=2.12.0,!=2.12.1,!=2.4.0,!=2.4.1,<3.0,>=2.0->pyiceberg>=0.7.1->bigframes) (0.4.2)\n", "Requirement already satisfied: pycparser in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from cffi>=2.0.0->cryptography>=38.0.3->google-auth<3.0,>=2.15.0->bigframes) (3.0)\n", "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/.venv/lib/python3.13/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib->gcsfs!=2025.5.0,!=2026.2.0,!=2026.3.0,>=2023.3.0->bigframes) (3.3.1)\n", "\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.1\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "%pip install --upgrade bigframes google-cloud-automl google-cloud-translate google-ai-generativelanguage tensorflow " ] }, { "cell_type": "markdown", "id": "c749e07c", "metadata": {}, "source": [ "**Important:** restart the kernel by going to \"Run -> Restart & clear cell outputs\" before continuing.\n", "\n", "Configure bigframes to use your GCP project. First, go to \"Add-ons -> Google Cloud SDK\" and click the \"Attach\" button. Then," ] }, { "cell_type": "code", "execution_count": 2, "id": "5e00777d", "metadata": { "execution": { "iopub.execute_input": "2025-08-18T20:17:14.873201Z", "iopub.status.busy": "2025-08-18T20:17:14.872905Z", "iopub.status.idle": "2025-08-18T20:17:14.946971Z", "shell.execute_reply": "2025-08-18T20:17:14.945996Z", "shell.execute_reply.started": "2025-08-18T20:17:14.873171Z" }, "trusted": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Not running on Kaggle, skipping Kaggle secrets initialization.\n" ] } ], "source": [ "try:\n", " from kaggle_secrets import UserSecretsClient\n", " user_secrets = UserSecretsClient()\n", " user_credential = user_secrets.get_gcloud_credential()\n", " user_secrets.set_tensorflow_credential(user_credential)\n", " print(\"Successfully authenticated using Kaggle secrets.\")\n", "except ImportError:\n", " print(\"Not running on Kaggle, skipping Kaggle secrets initialization.\")\n", "except Exception as e:\n", " print(f\"Could not initialize Kaggle secrets: {e}\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "b2e171de", "metadata": { "execution": { "iopub.execute_input": "2025-08-18T20:17:25.574192Z", "iopub.status.busy": "2025-08-18T20:17:25.573874Z", "iopub.status.idle": "2025-08-18T20:17:45.102002Z", "shell.execute_reply": "2025-08-18T20:17:45.101140Z", "shell.execute_reply.started": "2025-08-18T20:17:25.574168Z" }, "trusted": true }, "outputs": [], "source": [ "PROJECT = \"bigframes-dev\" # replace with your project. \n", "# Refer to https://cloud.google.com/bigquery/docs/multimodal-data-dataframes-tutorial#required_roles for your required permissions\n", "\n", "LOCATION = \"us\" # replace with your location.\n", "DATASET_ID = \"bigframes_samples\" # replace with your dataset ID.\n", "OUTPUT_BUCKET = \"bigframes_blob_test\" # replace with your GCS bucket. \n", "\n", "FULL_CONNECTION_ID = f\"{PROJECT}.{LOCATION}.bigframes-default-connection\"\n", "\n", "import bigframes\n", "# Setup project\n", "bigframes.options.bigquery.project = PROJECT\n", "bigframes.options.bigquery.location = LOCATION\n", "\n", "# Display options\n", "bigframes.options.display.blob_display_width = 300\n", "bigframes.options.display.progress_bar = None\n", "\n", "import bigframes.pandas as bpd\n", "import bigframes.bigquery as bbq\n", "\n", "def get_runtime_json_str(series, mode=\"R\", with_metadata=False):\n", " \"\"\"Get runtime JSON from objectref.\"\"\"\n", " s = bbq.obj.fetch_metadata(series) if with_metadata else series\n", " runtime = bbq.obj.get_access_url(s, mode=mode)\n", " return bbq.to_json_string(runtime)\n", "\n", "def get_metadata(series):\n", " metadata_obj = bbq.obj.fetch_metadata(series)\n", " return bbq.json_query(metadata_obj.struct.field(\"details\"), \"$.gcs_metadata\")\n", "\n", "def get_content_type(series):\n", " return bbq.json_value(get_metadata(series), \"$.content_type\")\n", "\n", "def get_size(series):\n", " return bbq.json_value(get_metadata(series), \"$.size\").astype(\"Int64\")\n", "\n", "def get_updated(series):\n", " return bpd.to_datetime(bbq.json_value(get_metadata(series), \"$.updated\").astype(\"Int64\"), unit=\"us\", utc=True)\n", "\n", "from IPython.display import HTML, display\n", "\n", "def render_images(df):\n", " \"\"\"Helper to display BigFrames DataFrame with rendered image previews.\"\"\"\n", " import bigframes.pandas as bpd\n", " import bigframes.bigquery as bbq\n", " import bigframes\n", " from bigframes import dtypes\n", " import json\n", " \n", " if isinstance(df, bpd.Series):\n", " df = df.to_frame()\n", " \n", " object_cols = [\n", " col for col, dtype in zip(df.columns, df.dtypes)\n", " if dtype == dtypes.OBJ_REF_DTYPE\n", " ]\n", " \n", " if not object_cols:\n", " display(df)\n", " return\n", "\n", " limit = bigframes.options.display.max_rows or 10\n", " view_df = df.head(limit)\n", " \n", " runtime_cols = {\n", " col: get_runtime_json_str(view_df[col], mode=\"R\", with_metadata=False) \n", " for col in object_cols\n", " }\n", " \n", " pandas_json_df = bpd.DataFrame(runtime_cols).to_pandas()\n", " final_pd = view_df.to_pandas()\n", " \n", " width = bigframes.options.display.blob_display_width or 300\n", " IMAGE_EXTENSIONS = (\".png\", \".jpg\", \".jpeg\", \".gif\", \".webp\")\n", " \n", " def format_cell_html(raw_json):\n", " if not raw_json:\n", " return \"\"\n", " try:\n", " obj_rt = json.loads(raw_json)\n", " if \"access_urls\" not in obj_rt:\n", " err = obj_rt.get(\"errors\", [{\"message\": \"URL Generation Failed\"}])[0].get(\"message\")\n", " return f'Error: {err}'\n", " \n", " uri = obj_rt.get(\"objectref\", {}).get(\"uri\", \"\")\n", " url = obj_rt[\"access_urls\"][\"read_url\"]\n", " \n", " if uri and str(uri).lower().endswith(IMAGE_EXTENSIONS):\n", " return f''\n", " \n", " return f'{uri if uri else \"view\"}'\n", " except:\n", " return \"Format Error\"\n", "\n", " for col in object_cols:\n", " final_pd[col] = pandas_json_df[col].map(format_cell_html)\n", " \n", " display(HTML(final_pd.to_html(escape=False)))" ] }, { "cell_type": "code", "execution_count": 4, "id": "d17afaf1", "metadata": { "execution": { "iopub.execute_input": "2025-08-18T20:17:45.103530Z", "iopub.status.busy": "2025-08-18T20:17:45.103249Z", "iopub.status.idle": "2025-08-18T20:17:47.424586Z", "shell.execute_reply": "2025-08-18T20:17:47.423762Z", "shell.execute_reply.started": "2025-08-18T20:17:45.103499Z" }, "trusted": true }, "outputs": [], "source": [ "import gcsfs\n", "import bigframes.bigquery as bbq\n", "\n", "# List files using gcsfs (public bucket)\n", "fs = gcsfs.GCSFileSystem(anon=True)\n", "uris = fs.glob(\"gs://cloud-samples-data/bigquery/tutorials/cymbal-pets/images/*\")\n", "\n", "# Ensure URIs have gs:// prefix\n", "uris = [u if u.startswith(\"gs://\") else f\"gs://{u}\" for u in uris]\n", "\n", "# Read the URIs into a BigQuery DataFrame using UNNEST\n", "# We take the first 5 for this example\n", "df_image = bpd.read_gbq(f\"SELECT uri FROM UNNEST({uris[:5]}) as uri\")\n", "\n", "# Create the object reference column\n", "df_image['image'] = bbq.obj.make_ref(df_image['uri'], authorizer=FULL_CONNECTION_ID)\n", "df_image = df_image[['image']]" ] }, { "cell_type": "code", "execution_count": 5, "id": "3e84b922", "metadata": { "execution": { "iopub.execute_input": "2025-08-18T20:17:47.425873Z", "iopub.status.busy": "2025-08-18T20:17:47.425578Z", "iopub.status.idle": "2025-08-18T20:18:07.919961Z", "shell.execute_reply": "2025-08-18T20:18:07.918942Z", "shell.execute_reply.started": "2025-08-18T20:17:47.425844Z" }, "trusted": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", "instead of using `db_dtypes` in the future when available in pandas\n", "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" ] }, { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
image
0
1
2
3
4
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Take only the 5 images to deal with. Preview the content of the Mutimodal DataFrame\n", "df_image = df_image.head(5)\n", "render_images(df_image)" ] }, { "cell_type": "markdown", "id": "b0eaa73c", "metadata": {}, "source": [ "# 2. Combine unstructured data with structured data\n", "\n", "Now you can put more information into the table to describe the files. Such as author info from inputs, or other metadata from the gcs object itself." ] }, { "cell_type": "code", "execution_count": 6, "id": "7d64fb54", "metadata": { "execution": { "iopub.execute_input": "2025-08-18T20:18:07.922593Z", "iopub.status.busy": "2025-08-18T20:18:07.921884Z", "iopub.status.idle": "2025-08-18T20:18:35.549725Z", "shell.execute_reply": "2025-08-18T20:18:35.548942Z", "shell.execute_reply.started": "2025-08-18T20:18:07.922551Z" }, "trusted": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", "instead of using `db_dtypes` in the future when available in pandas\n", "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" ] }, { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
imageauthorcontent_typesizeupdated
0aliceimage/png7157662025-03-20 17:44:38+00:00
1bobimage/png11674062025-03-20 17:44:38+00:00
2bobimage/png11508922025-03-20 17:44:39+00:00
3aliceimage/png17365332025-03-20 17:44:39+00:00
4bobimage/png4397402025-03-20 17:44:39+00:00
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Combine unstructured data with structured data\n", "df_image[\"author\"] = [\"alice\", \"bob\", \"bob\", \"alice\", \"bob\"] # type: ignore\n", "df_image[\"content_type\"] = get_content_type(df_image[\"image\"])\n", "df_image[\"size\"] = get_size(df_image[\"image\"])\n", "df_image[\"updated\"] = get_updated(df_image[\"image\"])\n", "render_images(df_image)" ] }, { "cell_type": "markdown", "id": "a23ef0e4", "metadata": {}, "source": [ "Then you can filter the rows based on the structured data. And for different content types, you can display them respectively or together." ] }, { "cell_type": "code", "execution_count": 7, "id": "ce102df0", "metadata": { "execution": { "iopub.execute_input": "2025-08-18T20:18:55.300314Z", "iopub.status.busy": "2025-08-18T20:18:55.299993Z", "iopub.status.idle": "2025-08-18T20:19:09.154492Z", "shell.execute_reply": "2025-08-18T20:19:09.153315Z", "shell.execute_reply.started": "2025-08-18T20:18:55.300289Z" }, "trusted": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", "instead of using `db_dtypes` in the future when available in pandas\n", "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" ] }, { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
imageauthorcontent_typesizeupdated
0aliceimage/png7157662025-03-20 17:44:38+00:00
3aliceimage/png17365332025-03-20 17:44:39+00:00
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# filter images and display, you can also display audio and video types\n", "filtered_df = df_image[df_image[\"author\"] == \"alice\"]\n", "render_images(filtered_df)" ] }, { "cell_type": "markdown", "id": "db2b3b12", "metadata": {}, "source": [ "# 3. Conduct image transformations\n", "\n", "BigFrames Multimodal DataFrame provides image(and other) transformation functions. Such as image_blur, image_resize and image_normalize. The output can be saved to GCS folders or to BQ as bytes." ] }, { "cell_type": "code", "execution_count": 8, "id": "283036f5", "metadata": { "execution": { "iopub.execute_input": "2025-08-18T20:19:22.950652Z", "iopub.status.busy": "2025-08-18T20:19:22.950277Z", "iopub.status.idle": "2025-08-18T20:31:51.799997Z", "shell.execute_reply": "2025-08-18T20:31:51.798840Z", "shell.execute_reply.started": "2025-08-18T20:19:22.950625Z" }, "trusted": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/pandas/__init__.py:211: PreviewWarning: udf is in preview.\n", " return global_session.with_default_session(\n", "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dataframe.py:4695: FunctionAxisOnePreviewWarning: DataFrame.apply with parameter axis=1 scenario is in preview.\n", " warnings.warn(msg, category=bfe.FunctionAxisOnePreviewWarning)\n", "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", "instead of using `db_dtypes` in the future when available in pandas\n", "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" ] }, { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
imageblurred
0
1
2
3
4
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "@bpd.udf(\n", " input_types=[str, str, int, int],\n", " output_type=str,\n", " dataset=DATASET_ID,\n", " name=\"image_blur_kaggle\",\n", " bigquery_connection=FULL_CONNECTION_ID,\n", " packages=[\"opencv-python-headless\", \"numpy\", \"requests\"],\n", ")\n", "def image_blur(src_rt: str, dst_rt: str, kx: int, ky: int) -> str:\n", " import json\n", " import cv2 as cv\n", " import numpy as np\n", " import requests\n", " \n", " src_obj = json.loads(src_rt)\n", " if \"access_urls\" not in src_obj:\n", " raise ValueError(f\"Missing 'access_urls' in source object. Response: {src_obj}\")\n", " src_url = src_obj[\"access_urls\"][\"read_url\"]\n", " \n", " response = requests.get(src_url, timeout=30)\n", " response.raise_for_status()\n", " \n", " img = cv.imdecode(np.frombuffer(response.content, np.uint8), cv.IMREAD_UNCHANGED)\n", " if img is None:\n", " raise ValueError(\"cv.imdecode failed\")\n", " \n", " img_blurred = cv.blur(img, ksize=(int(kx), int(ky)))\n", " success, encoded = cv.imencode(\".jpeg\", img_blurred)\n", " \n", " if not success:\n", " raise ValueError(\"cv.imencode failed\")\n", " \n", " if dst_rt: # GCS Output Mode\n", " dst_obj = json.loads(dst_rt)\n", " if \"access_urls\" not in dst_obj:\n", " raise ValueError(f\"Missing 'access_urls' in destination object. Response: {dst_obj}\")\n", " dst_url = dst_obj[\"access_urls\"][\"write_url\"]\n", " \n", " requests.put(dst_url, data=encoded.tobytes(), headers={\"Content-Type\": \"image/jpeg\"}, timeout=30).raise_for_status()\n", " return dst_obj[\"objectref\"][\"uri\"]\n", " return \"\"\n", "\n", "def apply_transformation(series, dst_folder, udf, *args, verbose=False):\n", " import os\n", " dst_folder = os.path.join(dst_folder, \"\")\n", " metadata = bbq.obj.fetch_metadata(series)\n", " current_uri = metadata.struct.field(\"uri\")\n", " dst_uri = current_uri.str.replace(r\"^.*\\/(.*)$\", rf\"{dst_folder}\\1\", regex=True)\n", " \n", " # Bypass synchronous validation via JSON initialization\n", " dst_blob_df = bpd.DataFrame({\"uri\": dst_uri})\n", " dst_blob_df[\"authorizer\"] = FULL_CONNECTION_ID\n", " dst_blob = bbq.obj.make_ref(bbq.to_json(bbq.struct(dst_blob_df)))\n", "\n", " df_transform = bpd.DataFrame({\n", " \"src_rt\": get_runtime_json_str(series, mode=\"R\"),\n", " \"dst_rt\": get_runtime_json_str(dst_blob, mode=\"RW\"),\n", " })\n", " res = df_transform[[\"src_rt\", \"dst_rt\"]].apply(udf, axis=1, args=args)\n", " \n", " if verbose:\n", " return res\n", " \n", " res_df = bpd.DataFrame({\"uri\": res})\n", " res_df[\"authorizer\"] = FULL_CONNECTION_ID\n", " return bbq.obj.make_ref(bbq.to_json(bbq.struct(res_df)))\n", "\n", "# Apply Blur Transformation\n", "df_image[\"blurred\"] = apply_transformation(\n", " df_image[\"image\"], f\"gs://{OUTPUT_BUCKET}/image_blur_transformed/\",\n", " image_blur, 20, 20\n", ")\n", "render_images(df_image[[\"image\", \"blurred\"]])" ] }, { "cell_type": "markdown", "id": "2d68a468", "metadata": {}, "source": [ "# 4. Use LLM models to ask questions and generate embeddings on images" ] }, { "cell_type": "code", "execution_count": 10, "id": "662054a0", "metadata": { "execution": { "iopub.execute_input": "2025-08-18T20:36:13.954686Z", "iopub.status.busy": "2025-08-18T20:36:13.954340Z", "iopub.status.idle": "2025-08-18T20:36:43.225449Z", "shell.execute_reply": "2025-08-18T20:36:43.224579Z", "shell.execute_reply.started": "2025-08-18T20:36:13.954661Z" }, "trusted": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/core/logging/log_adapter.py:183: FutureWarning: Since upgrading the default model can cause unintended breakages, the\n", "default model will be removed in BigFrames 3.0. Please supply an\n", "explicit model to avoid this message.\n", " return method(*args, **kwargs)\n", "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/session/__init__.py:437: FutureWarning: You are using the BigFrames session default connection: bigframes-\n", "default-connection, which can be different from the\n", "BigQuery project default connection. This default\n", "connection may change in the future.\n", " warnings.warn(msg, category=FutureWarning)\n" ] } ], "source": [ "from bigframes.ml import llm\n", "gemini = llm.GeminiTextGenerator()" ] }, { "cell_type": "code", "execution_count": 11, "id": "a31730ff", "metadata": { "execution": { "iopub.execute_input": "2025-08-18T20:36:43.227798Z", "iopub.status.busy": "2025-08-18T20:36:43.227457Z", "iopub.status.idle": "2025-08-18T20:37:25.238649Z", "shell.execute_reply": "2025-08-18T20:37:25.237623Z", "shell.execute_reply.started": "2025-08-18T20:36:43.227764Z" }, "trusted": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", "instead of using `db_dtypes` in the future when available in pandas\n", "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" ] }, { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ml_generate_text_llm_resultimage
0Please provide me with the picture! I need to see the image to tell you what the item is and what color the picture is.\\n
1To answer your question accurately, I need you to provide me with the picture you are referring to. Once you provide the picture, I can analyze it and tell you what item is in the picture and what color the picture is.
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Ask the same question on the images\n", "df_image = df_image.head(2)\n", "answer = gemini.predict(df_image, prompt=[\"what item is it?\", \"what color is the picture?\"])\n", "render_images(answer[[\"ml_generate_text_llm_result\", \"image\"]])" ] }, { "cell_type": "code", "execution_count": 12, "id": "f5d2a1ed", "metadata": { "execution": { "iopub.execute_input": "2025-08-18T20:37:25.239875Z", "iopub.status.busy": "2025-08-18T20:37:25.239607Z", "iopub.status.idle": "2025-08-18T20:37:25.263034Z", "shell.execute_reply": "2025-08-18T20:37:25.262002Z", "shell.execute_reply.started": "2025-08-18T20:37:25.239847Z" }, "trusted": true }, "outputs": [], "source": [ "# Ask different questions\n", "df_image[\"question\"] = [\"what item is it?\", \"what color is the picture?\"]" ] }, { "cell_type": "code", "execution_count": 13, "id": "fb67bf8e", "metadata": { "execution": { "iopub.execute_input": "2025-08-18T20:37:25.264585Z", "iopub.status.busy": "2025-08-18T20:37:25.264072Z", "iopub.status.idle": "2025-08-18T20:38:10.129667Z", "shell.execute_reply": "2025-08-18T20:38:10.128677Z", "shell.execute_reply.started": "2025-08-18T20:37:25.264518Z" }, "trusted": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", "instead of using `db_dtypes` in the future when available in pandas\n", "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n", "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", "instead of using `db_dtypes` in the future when available in pandas\n", "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" ] }, { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ml_generate_text_llm_resultimage
0The item is a glass aquarium.
1Dark brown
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "answer_alt = gemini.predict(df_image, prompt=[df_image[\"question\"], df_image[\"image\"]])\n", "render_images(answer_alt[[\"ml_generate_text_llm_result\", \"image\"]])" ] }, { "cell_type": "code", "execution_count": 14, "id": "0cf33170", "metadata": { "execution": { "iopub.execute_input": "2025-08-18T20:38:10.130851Z", "iopub.status.busy": "2025-08-18T20:38:10.130617Z", "iopub.status.idle": "2025-08-18T20:39:04.790416Z", "shell.execute_reply": "2025-08-18T20:39:04.789398Z", "shell.execute_reply.started": "2025-08-18T20:38:10.130833Z" }, "trusted": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/core/logging/log_adapter.py:183: FutureWarning: Since upgrading the default model can cause unintended breakages, the\n", "default model will be removed in BigFrames 3.0. Please supply an\n", "explicit model to avoid this message.\n", " return method(*args, **kwargs)\n", "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/session/__init__.py:437: FutureWarning: You are using the BigFrames session default connection: bigframes-\n", "default-connection, which can be different from the\n", "BigQuery project default connection. This default\n", "connection may change in the future.\n", " warnings.warn(msg, category=FutureWarning)\n", "/usr/local/google/home/shuowei/src/google-cloud-python/google-cloud-python/packages/bigframes/bigframes/dtypes.py:1044: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n", "instead of using `db_dtypes` in the future when available in pandas\n", "(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n", " warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ml_generate_embedding_resultml_generate_embedding_statusml_generate_embedding_start_secml_generate_embedding_end_seccontent
0[ 0.03416207 0.0419732 -0.0227391 ... -0.03...<NA><NA>{\"access_urls\":{\"expiry_time\":\"2026-05-02T03:3...
1[ 0.01908903 0.0193082 -0.00221754 ... 0.00...<NA><NA>{\"access_urls\":{\"expiry_time\":\"2026-05-02T03:3...
\n", "

2 rows × 5 columns

\n", "
[2 rows x 5 columns in total]" ], "text/plain": [ " ml_generate_embedding_result \\\n", "0 [ 0.03416207 0.0419732 -0.0227391 ... -0.03... \n", "1 [ 0.01908903 0.0193082 -0.00221754 ... 0.00... \n", "\n", " ml_generate_embedding_status ml_generate_embedding_start_sec \\\n", "0 \n", "1 \n", "\n", " ml_generate_embedding_end_sec \\\n", "0 \n", "1 \n", "\n", " content \n", "0 {\"access_urls\":{\"expiry_time\":\"2026-05-02T03:3... \n", "1 {\"access_urls\":{\"expiry_time\":\"2026-05-02T03:3... \n", "\n", "[2 rows x 5 columns]" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Generate embeddings.\n", "embed_model = llm.MultimodalEmbeddingGenerator()\n", "embeddings = embed_model.predict(df_image[\"image\"])\n", "embeddings" ] } ], "metadata": { "kaggle": { "accelerator": "none", "dataSources": [ { "databundleVersionId": 13391012, "sourceId": 110281, "sourceType": "competition" } ], "dockerImageVersionId": 31089, "isGpuEnabled": false, "isInternetEnabled": true, "language": "python", "sourceType": "notebook" }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.13" } }, "nbformat": 4, "nbformat_minor": 4 }