Transfer Learning with Keras Applications

Introduction

Transfer learning is a powerful technique where a model developed for one task is reused as the starting point for a model on a second task. It is especially popular in computer vision, where pre-trained models like ResNet50, which were trained on the massive ImageNet dataset, can be used as powerful, ready-made feature extractors.

The kerasnip package makes it easy to incorporate these pre-trained Keras Applications directly into a tidymodels workflow. This vignette will demonstrate how to:

Define a kerasnip model that uses a pre-trained ResNet50 as a frozen base layer.
Add a new, trainable classification “head” on top of the frozen base.
Tune the hyperparameters of the new classification head using a standard tidymodels workflow.

Setup

First, we load the necessary packages.

library(kerasnip)
library(tidymodels)
#> ── Attaching packages ────────────────────────────────────── tidymodels 1.4.1 ──
#> ✔ broom        1.0.10     ✔ recipes      1.3.1 
#> ✔ dials        1.4.2      ✔ rsample      1.3.1 
#> ✔ dplyr        1.1.4      ✔ tailor       0.1.0 
#> ✔ ggplot2      4.0.0      ✔ tidyr        1.3.1 
#> ✔ infer        1.0.9      ✔ tune         2.0.0 
#> ✔ modeldata    1.5.1      ✔ workflows    1.3.0 
#> ✔ parsnip      1.3.3      ✔ workflowsets 1.1.1 
#> ✔ purrr        1.1.0      ✔ yardstick    1.3.2
#> ── Conflicts ───────────────────────────────────────── tidymodels_conflicts() ──
#> ✖ purrr::discard() masks scales::discard()
#> ✖ dplyr::filter()  masks stats::filter()
#> ✖ dplyr::lag()     masks stats::lag()
#> ✖ recipes::step()  masks stats::step()
library(keras3)
#> 
#> Attaching package: 'keras3'
#> The following object is masked from 'package:yardstick':
#> 
#>     get_weights

Data Preparation

We’ll use the CIFAR-10 dataset, which consists of 60,000 32x32 color images in 10 classes. keras3 provides a convenient function to download it.

The ResNet50 model was pre-trained on ImageNet, which has a different set of classes. Our goal is to fine-tune it to classify the 10 classes in CIFAR-10.

# Load CIFAR-10 dataset
cifar10 <- dataset_cifar10()
#> Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz
#>         0/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0s/step    98304/170498071 ━━━━━━━━━━━━━━━━━━━━ 1:58 1us/step   671744/170498071 ━━━━━━━━━━━━━━━━━━━━ 29s 0us/step   2080768/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step  4644864/170498071 ━━━━━━━━━━━━━━━━━━━━ 7s 0us/step   6176768/170498071 ━━━━━━━━━━━━━━━━━━━━ 7s 0us/step  8257536/170498071 ━━━━━━━━━━━━━━━━━━━━ 6s 0us/step 10305536/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step 12271616/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step 12845056/170498071 ━━━━━━━━━━━━━━━━━━━━ 6s 0us/step 13524992/170498071 ━━━━━━━━━━━━━━━━━━━━ 6s 0us/step 13918208/170498071 ━━━━━━━━━━━━━━━━━━━━ 6s 0us/step 14155776/170498071 ━━━━━━━━━━━━━━━━━━━━ 7s 0us/step 14376960/170498071 ━━━━━━━━━━━━━━━━━━━━ 7s 0us/step 14606336/170498071 ━━━━━━━━━━━━━━━━━━━━ 8s 0us/step 14868480/170498071 ━━━━━━━━━━━━━━━━━━━━ 8s 0us/step 15081472/170498071 ━━━━━━━━━━━━━━━━━━━━ 8s 0us/step 15327232/170498071 ━━━━━━━━━━━━━━━━━━━━ 9s 0us/step 15605760/170498071 ━━━━━━━━━━━━━━━━━━━━ 9s 0us/step 15826944/170498071 ━━━━━━━━━━━━━━━━━━━━ 10s 0us/step 16072704/170498071 ━━━━━━━━━━━━━━━━━━━━ 10s 0us/step 16351232/170498071 ━━━━━━━━━━━━━━━━━━━━ 10s 0us/step 16613376/170498071 ━━━━━━━━━━━━━━━━━━━━ 10s 0us/step 16842752/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 17104896/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 17399808/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 17653760/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 17899520/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 18161664/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 18432000/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 18759680/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 19021824/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 19283968/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 19554304/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 19832832/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 20176896/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 20480000/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 20774912/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 21069824/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 21438464/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 21741568/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 22044672/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 22347776/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 22691840/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 23060480/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 23379968/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 23740416/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 24117248/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 24453120/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 24813568/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 25149440/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 25542656/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 25886720/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 26222592/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 26615808/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 27009024/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 27369472/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 27721728/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 28131328/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 28532736/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 28901376/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 29286400/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 29646848/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 30048256/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 30498816/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 30883840/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 31293440/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 31703040/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 32104448/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 32538624/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 32980992/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 33382400/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 33792000/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 34226176/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 34668544/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 35135488/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 35553280/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 36020224/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 36478976/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 36954112/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 37404672/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 37863424/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 38297600/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 38756352/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 38813696/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 40230912/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 40787968/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 41336832/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 41918464/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 42467328/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 42950656/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 43474944/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 43974656/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 44482560/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 44965888/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 45490176/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 46039040/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 46546944/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 47063040/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 47554560/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 47923200/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 48422912/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 48832512/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 49020928/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 49209344/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 49299456/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 49405952/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 49479680/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 49553408/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 49635328/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 49709056/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 49799168/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 49897472/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 49979392/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 50094080/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 50192384/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 50307072/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 50421760/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 50528256/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 50651136/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 50774016/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 50896896/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 51027968/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 51159040/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 51298304/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 51437568/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 51576832/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 51740672/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 51879936/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 52027392/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 52183040/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 52371456/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 52535296/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 52723712/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 52895744/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 53067776/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 53248000/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 53452800/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 53641216/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 53829632/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 54042624/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 54247424/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 54444032/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 54665216/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 54886400/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 55091200/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 55328768/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 55558144/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 55779328/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 56016896/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 56246272/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 56475648/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 56705024/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 56934400/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 57204736/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 57450496/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 57704448/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 57966592/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 58236928/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 58482688/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 58761216/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 59023360/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 59326464/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 59580416/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 59875328/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 60194816/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 60465152/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 60776448/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 61104128/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 61399040/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 61710336/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 62005248/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 62341120/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 62644224/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 62988288/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 63356928/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 63709184/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 64028672/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 64380928/170498071 ━━━━━━━━━━━━━━━━━━━━ 16s 0us/step 64716800/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 65085440/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 65413120/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 65781760/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 66207744/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 66535424/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 66887680/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 67305472/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 67674112/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 67952640/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 68288512/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 68616192/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 68952064/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 69296128/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 69632000/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 69976064/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 70328320/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 70664192/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 71008256/170498071 ━━━━━━━━━━━━━━━━━━━━ 15s 0us/step 71368704/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 71712768/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 72081408/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 72450048/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 72826880/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 73146368/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 73523200/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 73900032/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 74276864/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 74670080/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 75038720/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 75431936/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 75800576/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 76185600/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 76578816/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 76980224/170498071 ━━━━━━━━━━━━━━━━━━━━ 14s 0us/step 77357056/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 77742080/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 78143488/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 78536704/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 78929920/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 79331328/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 79724544/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 80134144/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 80543744/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 80961536/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 81395712/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 81805312/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 82247680/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 82690048/170498071 ━━━━━━━━━━━━━━━━━━━━ 13s 0us/step 83091456/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 83517440/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 83959808/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 84385792/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 84811776/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 85237760/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 85680128/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 86122496/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 86548480/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 86974464/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 87433216/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 87900160/170498071 ━━━━━━━━━━━━━━━━━━━━ 12s 0us/step 88383488/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 88842240/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 89350144/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 89833472/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 90365952/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 90890240/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 91447296/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 92004352/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 92536832/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 93093888/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 93634560/170498071 ━━━━━━━━━━━━━━━━━━━━ 11s 0us/step 94208000/170498071 ━━━━━━━━━━━━━━━━━━━━ 10s 0us/step 94797824/170498071 ━━━━━━━━━━━━━━━━━━━━ 10s 0us/step 95346688/170498071 ━━━━━━━━━━━━━━━━━━━━ 10s 0us/step 95911936/170498071 ━━━━━━━━━━━━━━━━━━━━ 10s 0us/step 96493568/170498071 ━━━━━━━━━━━━━━━━━━━━ 10s 0us/step 97075200/170498071 ━━━━━━━━━━━━━━━━━━━━ 10s 0us/step 97681408/170498071 ━━━━━━━━━━━━━━━━━━━━ 10s 0us/step 98230272/170498071 ━━━━━━━━━━━━━━━━━━━━ 10s 0us/step 98836480/170498071 ━━━━━━━━━━━━━━━━━━━━ 10s 0us/step 99409920/170498071 ━━━━━━━━━━━━━━━━━━━━ 9s 0us/step 100007936/170498071 ━━━━━━━━━━━━━━━━━━━━ 9s 0us/step100630528/170498071 ━━━━━━━━━━━━━━━━━━━━ 9s 0us/step101195776/170498071 ━━━━━━━━━━━━━━━━━━━━ 9s 0us/step101769216/170498071 ━━━━━━━━━━━━━━━━━━━━ 9s 0us/step102375424/170498071 ━━━━━━━━━━━━━━━━━━━━ 9s 0us/step103079936/170498071 ━━━━━━━━━━━━━━━━━━━━ 9s 0us/step103686144/170498071 ━━━━━━━━━━━━━━━━━━━━ 9s 0us/step104292352/170498071 ━━━━━━━━━━━━━━━━━━━━ 9s 0us/step104914944/170498071 ━━━━━━━━━━━━━━━━━━━━ 9s 0us/step105586688/170498071 ━━━━━━━━━━━━━━━━━━━━ 8s 0us/step106283008/170498071 ━━━━━━━━━━━━━━━━━━━━ 8s 0us/step106889216/170498071 ━━━━━━━━━━━━━━━━━━━━ 8s 0us/step107503616/170498071 ━━━━━━━━━━━━━━━━━━━━ 8s 0us/step108118016/170498071 ━━━━━━━━━━━━━━━━━━━━ 8s 0us/step108773376/170498071 ━━━━━━━━━━━━━━━━━━━━ 8s 0us/step109518848/170498071 ━━━━━━━━━━━━━━━━━━━━ 8s 0us/step110116864/170498071 ━━━━━━━━━━━━━━━━━━━━ 8s 0us/step110714880/170498071 ━━━━━━━━━━━━━━━━━━━━ 8s 0us/step111370240/170498071 ━━━━━━━━━━━━━━━━━━━━ 7s 0us/step112041984/170498071 ━━━━━━━━━━━━━━━━━━━━ 7s 0us/step112754688/170498071 ━━━━━━━━━━━━━━━━━━━━ 7s 0us/step113418240/170498071 ━━━━━━━━━━━━━━━━━━━━ 7s 0us/step114081792/170498071 ━━━━━━━━━━━━━━━━━━━━ 7s 0us/step114720768/170498071 ━━━━━━━━━━━━━━━━━━━━ 7s 0us/step115408896/170498071 ━━━━━━━━━━━━━━━━━━━━ 7s 0us/step116097024/170498071 ━━━━━━━━━━━━━━━━━━━━ 7s 0us/step116817920/170498071 ━━━━━━━━━━━━━━━━━━━━ 7s 0us/step117514240/170498071 ━━━━━━━━━━━━━━━━━━━━ 6s 0us/step118161408/170498071 ━━━━━━━━━━━━━━━━━━━━ 6s 0us/step118890496/170498071 ━━━━━━━━━━━━━━━━━━━━ 6s 0us/step119545856/170498071 ━━━━━━━━━━━━━━━━━━━━ 6s 0us/step120250368/170498071 ━━━━━━━━━━━━━━━━━━━━ 6s 0us/step121020416/170498071 ━━━━━━━━━━━━━━━━━━━━ 6s 0us/step121692160/170498071 ━━━━━━━━━━━━━━━━━━━━ 6s 0us/step122429440/170498071 ━━━━━━━━━━━━━━━━━━━━ 6s 0us/step123084800/170498071 ━━━━━━━━━━━━━━━━━━━━ 6s 0us/step123813888/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step124592128/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step125313024/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step125894656/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step126410752/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step126828544/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step127107072/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step127459328/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step127754240/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step127934464/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step128106496/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step128319488/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step128516096/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step128704512/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step128901120/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step129114112/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step129343488/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step129540096/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step129753088/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step129990656/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step130220032/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step130433024/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step130654208/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step130908160/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step131162112/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step131399680/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step131661824/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step131891200/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step132153344/170498071 ━━━━━━━━━━━━━━━━━━━━ 5s 0us/step132407296/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step132677632/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step132931584/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step133218304/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step133464064/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step133750784/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step134021120/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step134324224/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step134594560/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step134914048/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step135233536/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step135512064/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step135847936/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step136167424/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step136462336/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step136749056/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step137125888/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step137388032/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step137748480/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step138092544/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step138420224/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step138797056/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step139141120/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step139476992/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step139837440/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step140099584/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step140451840/170498071 ━━━━━━━━━━━━━━━━━━━━ 4s 0us/step140820480/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step141164544/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step141533184/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step141926400/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step142262272/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step142663680/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step143015936/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step143384576/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step143777792/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step144203776/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step144580608/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step144908288/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step145383424/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step145768448/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step146161664/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step146702336/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step147095552/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step147472384/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step147841024/170498071 ━━━━━━━━━━━━━━━━━━━━ 3s 0us/step148201472/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step148570112/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step148914176/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step149233664/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step149577728/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step149946368/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step150290432/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step150642688/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step150986752/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step151339008/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step151699456/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step152051712/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step152403968/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step152748032/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step153100288/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step153477120/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step153821184/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step154181632/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step154542080/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step154910720/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step155295744/170498071 ━━━━━━━━━━━━━━━━━━━━ 2s 0us/step155664384/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step156041216/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step156377088/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step156737536/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step157106176/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step157450240/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step157818880/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step158179328/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step158507008/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step158883840/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step159252480/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step159612928/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step159965184/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step160301056/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step160645120/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step160989184/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step161316864/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step161669120/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step162021376/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step162365440/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step162701312/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step163061760/170498071 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step163405824/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step163741696/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step164110336/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step164462592/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step164798464/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step165142528/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step165519360/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step165904384/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step166273024/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step166633472/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step167002112/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step167370752/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step167747584/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step168108032/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step168468480/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step168853504/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step169222144/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step169598976/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step169975808/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step170344448/170498071 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step170498071/170498071 ━━━━━━━━━━━━━━━━━━━━ 23s 0us/step

# Separate training and test data
x_train <- cifar10$train$x
y_train <- cifar10$train$y
x_test <- cifar10$test$x
y_test <- cifar10$test$y

# Rescale pixel values from [0, 255] to [0, 1]
x_train <- x_train / 255
x_test <- x_test / 255

# Convert outcomes to factors for tidymodels
y_train_factor <- factor(y_train[, 1])
y_test_factor <- factor(y_test[, 1])

# For tidymodels, it's best to work with data frames.
# We'll use a list-column to hold the image arrays.
train_df <- tibble::tibble(
  x = lapply(seq_len(nrow(x_train)), function(i) x_train[i, , , , drop = TRUE]),
  y = y_train_factor
)

test_df <- tibble::tibble(
  x = lapply(seq_len(nrow(x_test)), function(i) x_test[i, , , , drop = TRUE]),
  y = y_test_factor
)

# Use a smaller subset for faster vignette execution
train_df_small <- train_df[1:500, ]
test_df_small <- test_df[1:100, ]

Functional API with a Pre-trained Base

The standard approach for transfer learning is to use the Keras Functional API. We will define a model where: 1. The base is a pre-trained ResNet50, with its final classification layer removed (include_top = FALSE). 2. The weights of the base are frozen (trainable = FALSE) so that only our new layers are trained. 3. A new classification “head” is added, consisting of a flatten layer and a dense output layer.

Define Layer Blocks

# Input block: shape is determined automatically from the data
input_block <- function(input_shape) {
  layer_input(shape = input_shape)
}

# ResNet50 base block
resnet_base_block <- function(tensor) {
  # The base model is not trainable; we use it for feature extraction.
  resnet_base <- application_resnet50(
    weights = "imagenet",
    include_top = FALSE
  )
  resnet_base$trainable <- FALSE
  resnet_base(tensor)
}

# New classification head
flatten_block <- function(tensor) {
  tensor |> layer_flatten()
}

output_block_functional <- function(tensor, num_classes) {
  tensor |> layer_dense(units = num_classes, activation = "softmax")
}

Create the `kerasnip` Specification

We connect these blocks using create_keras_functional_spec().

create_keras_functional_spec(
  model_name = "resnet_transfer",
  layer_blocks = list(
    input = input_block,
    resnet_base = inp_spec(resnet_base_block, "input"),
    flatten = inp_spec(flatten_block, "resnet_base"),
    output = inp_spec(output_block_functional, "flatten")
  ),
  mode = "classification"
)

Fit and Evaluate the Model

Now we can use our new resnet_transfer() specification within a tidymodels workflow.

spec_functional <- resnet_transfer(
  fit_epochs = 5,
  fit_validation_split = 0.2
) |>
  set_engine("keras")

rec_functional <- recipe(y ~ x, data = train_df_small)

wf_functional <- workflow() |>
  add_recipe(rec_functional) |>
  add_model(spec_functional)

fit_functional <- fit(wf_functional, data = train_df_small)
#> Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5
#>        0/94765736 ━━━━━━━━━━━━━━━━━━━━ 0s 0s/step 4022272/94765736 ━━━━━━━━━━━━━━━━━━━━ 1s 0us/step21250048/94765736 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step35774464/94765736 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step51068928/94765736 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step68583424/94765736 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step86335488/94765736 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step94765736/94765736 ━━━━━━━━━━━━━━━━━━━━ 0s 0us/step

# Evaluate on the test set
predictions <- predict(fit_functional, new_data = test_df_small)
#> 4/4 - 2s - 525ms/step
bind_cols(predictions, test_df_small) |>
  accuracy(truth = y, estimate = .pred_class)
#> # A tibble: 1 × 3
#>   .metric  .estimator .estimate
#>   <chr>    <chr>          <dbl>
#> 1 accuracy multiclass       0.2

Even with a small dataset and few epochs, the pre-trained features from ResNet50 give us a reasonable starting point for accuracy.

Conclusion

This vignette demonstrated how kerasnip bridges the world of pre-trained Keras applications with the structured, reproducible workflows of tidymodels.

The Functional API is the most direct way to perform transfer learning by attaching a new head to a frozen base model.

This approach allows you to leverage the power of deep learning models that have been trained on massive datasets, significantly boosting performance on smaller, domain-specific tasks.