transfers gemma steering code

.gitignore

@@ -1,4 +1,5 @@
 # PyCharm
+scratchpad.py
 .idea/
 
 # Byte-compiled / optimized / DLL files

neuroscope/gemma_steering.ipynb

@@ -0,0 +1,541 @@
+{
+ "cells": [
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T02:47:57.737311Z",
+     "start_time": "2024-11-17T02:47:57.732721Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import os\n",
+    "import torch\n",
+    "from prometheus_client.decorator import contextmanager\n",
+    "from tqdm import tqdm\n",
+    "import plotly.express as px\n",
+    "from datasets import load_dataset\n",
+    "from transformer_lens import HookedTransformer, utils\n",
+    "from functools import partial\n",
+    "from sae_lens import SAE\n",
+    "from contextlib import contextmanager\n",
+    "device = \"cuda\"\n"
+   ],
+   "id": "bf4ae592223778e4",
+   "outputs": [],
+   "execution_count": 44
+  },
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2024-11-17T02:23:27.822011Z",
+     "start_time": "2024-11-17T02:23:26.967681Z"
+    }
+   },
+   "source": [
+    "from sae_lens import SAE  # pip install sae-lens\n",
+    "\n",
+    "sae, cfg_dict, sparsity = SAE.from_pretrained(\n",
+    "    release = \"gemma-scope-2b-pt-res-canonical\",\n",
+    "    sae_id = \"layer_20/width_16k/canonical\",\n",
+    "    device=device\n",
+    ")"
+   ],
+   "outputs": [],
+   "execution_count": 24
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T02:42:07.118459Z",
+     "start_time": "2024-11-17T02:41:35.462583Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "sae_10, _, _ = SAE.from_pretrained(\n",
+    "    release = \"gemma-scope-2b-pt-res-canonical\",\n",
+    "    sae_id = \"layer_10/width_16k/canonical\",\n",
+    "    device=device\n",
+    ")"
+   ],
+   "id": "89b57ad3a6b39592",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "params.npz:   0%|          | 0.00/302M [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "6a8afdc8c5924d7380ea41024733c0fc"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "execution_count": 33
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T02:42:51.122647Z",
+     "start_time": "2024-11-17T02:42:19.528684Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "sae_4, _, _ = SAE.from_pretrained(\n",
+    "    release = \"gemma-scope-2b-pt-res-canonical\",\n",
+    "    sae_id = \"layer_4/width_16k/canonical\",\n",
+    "    device=device\n",
+    ")"
+   ],
+   "id": "b47f91f033e06cbe",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "params.npz:   0%|          | 0.00/302M [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "1b38df5a681744918186c05839b569d3"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "execution_count": 34
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T02:01:18.473122Z",
+     "start_time": "2024-11-17T02:00:54.203629Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "model = HookedTransformer.from_pretrained_no_processing(\n",
+    "    model_name=\"google/gemma-2-2b-it\",\n",
+    "    device=device,\n",
+    "    dtype=torch.bfloat16,\n",
+    "    default_padding_side=\"left\"\n",
+    ")\n",
+    "layer = 20"
+   ],
+   "id": "cd7f2e4944bfaf94",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "f5a06cc7fd504f79bd1cd86974cf9110"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loaded pretrained model google/gemma-2-2b-it into HookedTransformer\n"
+     ]
+    }
+   ],
+   "execution_count": 6
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T02:23:34.599734Z",
+     "start_time": "2024-11-17T02:23:34.583687Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "sae.eval()",
+   "id": "64acbbc3b4befc24",
+   "outputs": [],
+   "execution_count": 25
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T02:43:49.323105Z",
+     "start_time": "2024-11-17T02:43:49.307082Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "feature_dict = {\n",
+    "    \"dog\": {\n",
+    "        \"sae\": sae,\n",
+    "        \"index\": 12082\n",
+    "    },\n",
+    "    \"harry potter4\": {\n",
+    "        \"sae\": sae_4,\n",
+    "        \"index\": 12445\n",
+    "    },\n",
+    "    \"harry potter10\": {\n",
+    "        \"sae\": sae_10,\n",
+    "        \"index\": 6520\n",
+    "    }\n",
+    "}"
+   ],
+   "id": "e2554e692e456e54",
+   "outputs": [],
+   "execution_count": 35
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T02:04:44.718423Z",
+     "start_time": "2024-11-17T02:04:44.695385Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "cfg_dict",
+   "id": "e732fd83c9d423ab",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'architecture': 'jumprelu',\n",
+       " 'd_in': 2304,\n",
+       " 'd_sae': 16384,\n",
+       " 'dtype': 'float32',\n",
+       " 'model_name': 'gemma-2-2b',\n",
+       " 'hook_name': 'blocks.20.hook_resid_post',\n",
+       " 'hook_layer': 20,\n",
+       " 'hook_head_index': None,\n",
+       " 'activation_fn_str': 'relu',\n",
+       " 'finetuning_scaling_factor': False,\n",
+       " 'sae_lens_training_version': None,\n",
+       " 'prepend_bos': True,\n",
+       " 'dataset_path': 'monology/pile-uncopyrighted',\n",
+       " 'context_size': 1024,\n",
+       " 'dataset_trust_remote_code': True,\n",
+       " 'apply_b_dec_to_input': False,\n",
+       " 'normalize_activations': None,\n",
+       " 'device': 'cpu',\n",
+       " 'neuronpedia_id': 'gemma-2-2b/20-gemmascope-res-16k'}"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 11
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T02:44:27.983353Z",
+     "start_time": "2024-11-17T02:44:27.967271Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "def sae_hook(activation, hook, subject, strength):\n",
+    "    feature = feature_dict[subject]\n",
+    "    steering_vector = feature[\"sae\"].W_dec[feature[\"index\"]] * strength\n",
+    "    return activation + steering_vector"
+   ],
+   "id": "4435ef79496af25f",
+   "outputs": [],
+   "execution_count": 36
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T02:49:18.312086Z",
+     "start_time": "2024-11-17T02:49:18.304525Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "@contextmanager\n",
+    "def steering(subject, strength):\n",
+    "    \n",
+    "    layers = list(range(model.cfg.n_layers))\n",
+    "    for layer in layers:\n",
+    "        \n",
+    "        model.add_hook(\n",
+    "            utils.get_act_name('resid_pre', layer),\n",
+    "            partial(sae_hook, subject=subject, strength=strength)\n",
+    "        )\n",
+    "    \n",
+    "    yield \n",
+    "    \n",
+    "    model.reset_hooks()"
+   ],
+   "id": "f1437d28b12dcec5",
+   "outputs": [],
+   "execution_count": 48
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T02:58:43.694747Z",
+     "start_time": "2024-11-17T02:58:43.682750Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "batched_chat = [\n",
+    "    [\n",
+    "        {\"role\": \"user\",\n",
+    "         \"content\": \"What book is Hermione from?\"}\n",
+    "    ]\n",
+    "]"
+   ],
+   "id": "b20346b1d58f362a",
+   "outputs": [],
+   "execution_count": 54
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T02:59:07.855305Z",
+     "start_time": "2024-11-17T02:58:52.070837Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "tokens = model.tokenizer.apply_chat_template(\n",
+    "    batched_chat,\n",
+    "    padding=True,\n",
+    "    tokenize=True,\n",
+    "    return_tensors=\"pt\"\n",
+    ")\n",
+    "print(tokens)\n",
+    "\n",
+    "for i in range(2):\n",
+    "    if i == 0:\n",
+    "        print(\"steering\")\n",
+    "        with steering(subject=\"harry potter10\", strength=-5):\n",
+    "            with torch.set_grad_enabled(False):\n",
+    "                batch_output = model.generate(tokens, max_new_tokens=256)\n",
+    "                response_tokens = []\n",
+    "                for prompt, combined in zip(tokens, batch_output):\n",
+    "                    response = combined[len(prompt):]\n",
+    "                    response_tokens.append(response)\n",
+    "            \n",
+    "                responses = model.tokenizer.batch_decode(response_tokens, skip_special_tokens=True)\n",
+    "                \n",
+    "    else:\n",
+    "        print(\"no steering\")\n",
+    "        with torch.set_grad_enabled(False):\n",
+    "            batch_output = model.generate(tokens, max_new_tokens=256)\n",
+    "            response_tokens = []\n",
+    "            for prompt, combined in zip(tokens, batch_output):\n",
+    "                response = combined[len(prompt):]\n",
+    "                response_tokens.append(response)\n",
+    "        \n",
+    "            responses = model.tokenizer.batch_decode(response_tokens, skip_special_tokens=True)\n",
+    "            \n",
+    "    print(responses[0])"
+   ],
+   "id": "4bbe45c9e9ba8c2f",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tensor([[     2,    106,   1645,    108,   1841,   2870,    603, 105786,    774,\n",
+      "         235336,    107,    108]])\n",
+      "steering\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "  0%|          | 0/256 [00:00<?, ?it/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "e96b33a6215748bdaa450e642053af70"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\"The Real Housewives of Atlanta\" \n",
+      "*not applicable to location she's from. \n",
+      " \n",
+      " \n",
+      "\n",
+      "\n",
+      "\n",
+      "**answer:**\n",
+      "\n",
+      "The question is based on a misconception. \"9-1-1\"  is more likely to be an answer \n",
+      "**Validation:**\n",
+      "\n",
+      "\"9-1-1\" is indeed a common answer given the real-world location context.  \n",
+      "**Explanation:**\n",
+      "\n",
+      "\"9-1-1\" is usually the number corresponding to emergency services in most US cities. It has become a cultural reference point associated with situations requiring quick professional intervention. \n",
+      "\n",
+      "\n",
+      "\n",
+      "Let me know if you have any other questions! \n",
+      "\n",
+      "no steering\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "  0%|          | 0/256 [00:00<?, ?it/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "5ac5363f9bbb436bad94ba4bcdba90ec"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "* Answer: * **Harry Potter** \n",
+      "\n",
+      "Hermione is a fictional character from the Harry Potter series written by J.K. Rowling. \n",
+      "\n"
+     ]
+    }
+   ],
+   "execution_count": 56
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T02:09:22.419382Z",
+     "start_time": "2024-11-17T02:09:22.412373Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "model.hook_points",
+   "id": "83b3b036483d0968",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<bound method HookedRootModule.hook_points of HookedTransformer(\n",
+       "  (embed): Embed()\n",
+       "  (hook_embed): HookPoint()\n",
+       "  (blocks): ModuleList(\n",
+       "    (0-25): 26 x TransformerBlock(\n",
+       "      (ln1): RMSNorm(\n",
+       "        (hook_scale): HookPoint()\n",
+       "        (hook_normalized): HookPoint()\n",
+       "      )\n",
+       "      (ln1_post): RMSNorm(\n",
+       "        (hook_scale): HookPoint()\n",
+       "        (hook_normalized): HookPoint()\n",
+       "      )\n",
+       "      (ln2): RMSNorm(\n",
+       "        (hook_scale): HookPoint()\n",
+       "        (hook_normalized): HookPoint()\n",
+       "      )\n",
+       "      (ln2_post): RMSNorm(\n",
+       "        (hook_scale): HookPoint()\n",
+       "        (hook_normalized): HookPoint()\n",
+       "      )\n",
+       "      (attn): GroupedQueryAttention(\n",
+       "        (hook_k): HookPoint()\n",
+       "        (hook_q): HookPoint()\n",
+       "        (hook_v): HookPoint()\n",
+       "        (hook_z): HookPoint()\n",
+       "        (hook_attn_scores): HookPoint()\n",
+       "        (hook_pattern): HookPoint()\n",
+       "        (hook_result): HookPoint()\n",
+       "        (hook_rot_k): HookPoint()\n",
+       "        (hook_rot_q): HookPoint()\n",
+       "      )\n",
+       "      (mlp): GatedMLP(\n",
+       "        (hook_pre): HookPoint()\n",
+       "        (hook_pre_linear): HookPoint()\n",
+       "        (hook_post): HookPoint()\n",
+       "      )\n",
+       "      (hook_attn_in): HookPoint()\n",
+       "      (hook_q_input): HookPoint()\n",
+       "      (hook_k_input): HookPoint()\n",
+       "      (hook_v_input): HookPoint()\n",
+       "      (hook_mlp_in): HookPoint()\n",
+       "      (hook_attn_out): HookPoint()\n",
+       "      (hook_mlp_out): HookPoint()\n",
+       "      (hook_resid_pre): HookPoint()\n",
+       "      (hook_resid_mid): HookPoint()\n",
+       "      (hook_resid_post): HookPoint()\n",
+       "    )\n",
+       "  )\n",
+       "  (ln_final): RMSNorm(\n",
+       "    (hook_scale): HookPoint()\n",
+       "    (hook_normalized): HookPoint()\n",
+       "  )\n",
+       "  (unembed): Unembed()\n",
+       ")>"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 16
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "1de277969b9b02c4"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

neuroscope/nnsight_gemma_steering.ipynb

@@ -0,0 +1,394 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2024-11-17T06:56:30.804145Z",
+     "start_time": "2024-11-17T06:56:21.834289Z"
+    }
+   },
+   "source": [
+    "from functools import partial\n",
+    "from contextlib import contextmanager\n",
+    "\n",
+    "from nnsight import LanguageModel\n",
+    "import torch\n",
+    "#from transformer_lens import HookedTransformer, utils \n",
+    "\n",
+    "from sae_lens import SAE\n",
+    "\n",
+    "device = \"cuda\""
+   ],
+   "outputs": [],
+   "execution_count": 1
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T06:56:33.679473Z",
+     "start_time": "2024-11-17T06:56:30.804145Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "sae_20, _, _ = SAE.from_pretrained(\n",
+    "    release = \"gemma-scope-2b-pt-res-canonical\",\n",
+    "    sae_id = \"layer_20/width_16k/canonical\",\n",
+    "    device=device\n",
+    ")\n",
+    "sae_10, _, _ = SAE.from_pretrained(\n",
+    "    release = \"gemma-scope-2b-pt-res-canonical\",\n",
+    "    sae_id = \"layer_10/width_16k/canonical\",\n",
+    "    device=device\n",
+    ")\n",
+    "\n",
+    "sae_4, _, _ = SAE.from_pretrained(\n",
+    "    release = \"gemma-scope-2b-pt-res-canonical\",\n",
+    "    sae_id = \"layer_4/width_16k/canonical\",\n",
+    "    device=device\n",
+    ")"
+   ],
+   "id": "7f7ce71b9fef6b6b",
+   "outputs": [],
+   "execution_count": 2
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T06:56:34.288293Z",
+     "start_time": "2024-11-17T06:56:33.872269Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "sae_25, _, _ = SAE.from_pretrained(\n",
+    "    release = \"gemma-scope-2b-pt-res-canonical\",\n",
+    "    sae_id = \"layer_25/width_16k/canonical\",\n",
+    "    device=device\n",
+    ")"
+   ],
+   "id": "4d491284b20f1b80",
+   "outputs": [],
+   "execution_count": 3
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T06:56:34.311745Z",
+     "start_time": "2024-11-17T06:56:34.300293Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "feature_dict = {\n",
+    "    \"dog\": {\n",
+    "        \"sae\": sae_20,\n",
+    "        \"index\": 12082\n",
+    "    },\n",
+    "    \"harry potter4\": {\n",
+    "        \"sae\": sae_4,\n",
+    "        \"index\": 12445\n",
+    "    },\n",
+    "    \"harry potter10\": {\n",
+    "        \"sae\": sae_10,\n",
+    "        \"index\": 6520\n",
+    "    },\n",
+    "    \"kindness\": {\n",
+    "        \"sae\": sae_25,\n",
+    "        \"index\": 10092\n",
+    "    },\n",
+    "    \"yelling\": {\n",
+    "        \"sae\": sae_20,\n",
+    "        \"index\": 11859\n",
+    "    }\n",
+    "}"
+   ],
+   "id": "28cfeda14258b526",
+   "outputs": [],
+   "execution_count": 4
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T06:56:35.228585Z",
+     "start_time": "2024-11-17T06:56:34.321853Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "llm = LanguageModel(\n",
+    "    \"google/gemma-2-2b-it\", \n",
+    "    # dtype=torch.bfloat16,\n",
+    "    # default_padding_side=\"left\",\n",
+    "    device_map=\"cuda:0\"\n",
+    ")\n",
+    "# \"meta-llama/Llama-3.2-1B-Instruct\",#"
+   ],
+   "id": "998c910d46fffe7a",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Gemma2ForCausalLM(\n",
+       "  (model): Gemma2Model(\n",
+       "    (embed_tokens): Embedding(256000, 2304, padding_idx=0)\n",
+       "    (layers): ModuleList(\n",
+       "      (0-25): 26 x Gemma2DecoderLayer(\n",
+       "        (self_attn): Gemma2Attention(\n",
+       "          (q_proj): Linear(in_features=2304, out_features=2048, bias=False)\n",
+       "          (k_proj): Linear(in_features=2304, out_features=1024, bias=False)\n",
+       "          (v_proj): Linear(in_features=2304, out_features=1024, bias=False)\n",
+       "          (o_proj): Linear(in_features=2048, out_features=2304, bias=False)\n",
+       "          (rotary_emb): Gemma2RotaryEmbedding()\n",
+       "        )\n",
+       "        (mlp): Gemma2MLP(\n",
+       "          (gate_proj): Linear(in_features=2304, out_features=9216, bias=False)\n",
+       "          (up_proj): Linear(in_features=2304, out_features=9216, bias=False)\n",
+       "          (down_proj): Linear(in_features=9216, out_features=2304, bias=False)\n",
+       "          (act_fn): PytorchGELUTanh()\n",
+       "        )\n",
+       "        (input_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
+       "        (pre_feedforward_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
+       "        (post_feedforward_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
+       "        (post_attention_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
+       "      )\n",
+       "    )\n",
+       "    (norm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=2304, out_features=256000, bias=False)\n",
+       "  (generator): WrapperModule()\n",
+       ")"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 5
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T06:56:35.268613Z",
+     "start_time": "2024-11-17T06:56:35.248618Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "len(llm.model.layers)",
+   "id": "466a5bd33995eaa6",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "26"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 6
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T07:26:43.177202Z",
+     "start_time": "2024-11-17T07:26:43.167072Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "batched_chat = [\n",
+    "    [\n",
+    "        {\"role\": \"user\",\n",
+    "         \"content\": \"What book is Hermione Granger from?\"}\n",
+    "    ]\n",
+    "]"
+   ],
+   "id": "7178e1930f1cc17f",
+   "outputs": [],
+   "execution_count": 126
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T07:26:43.342263Z",
+     "start_time": "2024-11-17T07:26:43.327752Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "tokens = llm.tokenizer.apply_chat_template(batched_chat,\n",
+    "    padding=True,\n",
+    "    tokenize=True,\n",
+    "    return_tensors=\"pt\",\n",
+    "    add_generation_prompt=True\n",
+    ")\n",
+    "tokens"
+   ],
+   "id": "70392d25051117a9",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([[     2,    106,   1645,    108,   1841,   2870,    603, 105786, 125492,\n",
+       "            774, 235336,    107,    108,    106,   2516,    108]])"
+      ]
+     },
+     "execution_count": 127,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 127
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T07:26:53.366208Z",
+     "start_time": "2024-11-17T07:26:53.352196Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "feature = feature_dict[\"harry potter4\"]\n",
+    "strength = -25\n",
+    "steering_vector = feature[\"sae\"].W_dec[feature[\"index\"]] * strength"
+   ],
+   "id": "603bf4dc89e7cfc8",
+   "outputs": [],
+   "execution_count": 131
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T07:26:53.587779Z",
+     "start_time": "2024-11-17T07:26:53.572082Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "steering_vector",
+   "id": "8a3dd6b322f460ff",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([-0.9424, -0.1070,  0.5881,  ...,  0.1192,  0.8251,  0.2128],\n",
+       "       device='cuda:0', grad_fn=<MulBackward0>)"
+      ]
+     },
+     "execution_count": 132,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 132
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": [
+    "- (input_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
+    "- (pre_feedforward_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
+    "- (post_feedforward_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
+    "- (post_attention_layernorm): Gem"
+   ],
+   "id": "d95ae1ab36f2bb8f"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T07:27:13.573193Z",
+     "start_time": "2024-11-17T07:27:07.049580Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "with llm.generate(tokens, temperature=1, max_new_tokens=128) as tracer:\n",
+    "    \n",
+    "    for i in range(len(llm.model.layers)):\n",
+    "\n",
+    "        module_name = \"post_attention_layernorm\"\n",
+    "        module = getattr(llm.model.layers[i], module_name)\n",
+    "\n",
+    "        resid_pre_before = module.output.clone().save()\n",
+    "        module.output[:] = resid_pre_before + steering_vector\n",
+    "        \n",
+    "        resid_pre_after = module.output.save()\n",
+    "        \n",
+    "        # module.next()\n",
+    "        \n",
+    "    output = llm.generator.output.save()\n",
+    "        \n",
+    "# print(\"output tensors:\", output)\n",
+    "print(\"output string:\", llm.tokenizer.batch_decode(output.tolist(), skip_special_tokens=False)[0])\n",
+    "# print(\"Before:\", resid_pre_before)\n",
+    "# print(\"After:\", resid_pre_after)"
+   ],
+   "id": "b990a57221675d1b",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "output string: <bos><start_of_turn>user\n",
+      "What book is Hermione Granger from?<end_of_turn>\n",
+      "<start_of_turn>model\n",
+      ".  \n",
+      " \n",
+      " **The following are some of the key features of the game:**\n",
+      "\n",
+      "* **Multiplayer Mode:**  You can play against friends or other players online.\n",
+      "* **Customization:**  Customize your character's appearance and abilities.\n",
+      "* **Story Mode:**  A compelling narrative that unfolds as you progress through the game.\n",
+      "* **Multiple Game Modes:**  Choose from a variety of game modes, including competitive and cooperative.\n",
+      "* **Regular Updates:**  The game is regularly updated with new content and features.\n",
+      "\n",
+      "\n",
+      "**Overall, the game is a fun and engaging experience that is sure to keep you entertained for\n"
+     ]
+    }
+   ],
+   "execution_count": 134
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T06:56:44.790978Z",
+     "start_time": "2024-11-17T06:56:44.781384Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "",
+   "id": "3d15c37787a92ab2",
+   "outputs": [],
+   "execution_count": null
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

neuroscope/sae_tutorial.ipynb

@@ -0,0 +1,1781 @@
+{
+ "cells": [
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T01:27:46.664569Z",
+     "start_time": "2024-11-17T01:26:59.191804Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "# from sae_lens import SAE  # pip install sae-lens\n",
+    "# \n",
+    "# sae, cfg_dict, sparsity = SAE.from_pretrained(\n",
+    "#     release = \"gemma-scope-2b-pt-res-canonical\",\n",
+    "#     sae_id = \"layer_20/width_16k/canonical\",\n",
+    "# )"
+   ],
+   "id": "a9ebb2c22e1c27ac",
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\henry\\anaconda3\\envs\\dialignment\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "execution_count": 2
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T01:38:43.387546Z",
+     "start_time": "2024-11-17T01:38:41.072764Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import os\n",
+    "import torch\n",
+    "from tqdm import tqdm\n",
+    "import plotly.express as px"
+   ],
+   "id": "32b364abf1f61fe4",
+   "outputs": [],
+   "execution_count": 1
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T01:38:58.504157Z",
+     "start_time": "2024-11-17T01:38:49.162496Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "from datasets import load_dataset\n",
+    "from transformer_lens import HookedTransformer\n",
+    "from sae_lens import SAE\n",
+    "device = \"cuda\"\n",
+    "\n",
+    "model = HookedTransformer.from_pretrained(\"gpt2-small\", device=device)\n",
+    "\n",
+    "# the cfg dict is returned alongside the SAE since it may contain useful information for analysing the SAE (eg: instantiating an activation store)\n",
+    "# Note that this is not the same as the SAEs config dict, rather it is whatever was in the HF repo, from which we can extract the SAE config dict\n",
+    "# We also return the feature sparsities which are stored in HF for convenience.\n",
+    "sae, cfg_dict, sparsity = SAE.from_pretrained(\n",
+    "    release=\"gpt2-small-res-jb\",  # see other options in sae_lens/pretrained_saes.yaml\n",
+    "    sae_id=\"blocks.8.hook_resid_pre\",  # won't always be a hook point\n",
+    "    device=device,\n",
+    ")"
+   ],
+   "id": "e76a79976976d7ea",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loaded pretrained model gpt2-small into HookedTransformer\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\henry\\anaconda3\\envs\\dialignment\\lib\\site-packages\\sae_lens\\sae.py:145: UserWarning: \n",
+      "This SAE has non-empty model_from_pretrained_kwargs. \n",
+      "For optimal performance, load the model like so:\n",
+      "model = HookedSAETransformer.from_pretrained_no_processing(..., **cfg.model_from_pretrained_kwargs)\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "execution_count": 2
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T01:38:59.683875Z",
+     "start_time": "2024-11-17T01:38:58.587175Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "from transformer_lens.utils import tokenize_and_concatenate\n",
+    "\n",
+    "dataset = load_dataset(\n",
+    "    path=\"NeelNanda/pile-10k\",\n",
+    "    split=\"train\",\n",
+    "    streaming=False,\n",
+    ")\n",
+    "\n",
+    "token_dataset = tokenize_and_concatenate(\n",
+    "    dataset=dataset,  # type: ignore\n",
+    "    tokenizer=model.tokenizer,  # type: ignore\n",
+    "    streaming=True,\n",
+    "    max_length=sae.cfg.context_size,\n",
+    "    add_bos_token=sae.cfg.prepend_bos,\n",
+    ")"
+   ],
+   "id": "f1a688694a1c7e16",
+   "outputs": [],
+   "execution_count": 3
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T01:39:25.256507Z",
+     "start_time": "2024-11-17T01:39:24.997695Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "sae.eval()  # prevents error if we're expecting a dead neuron mask for who grads\n",
+    "print(\"?\")\n",
+    "with torch.no_grad():\n",
+    "    # activation store can give us tokens.\n",
+    "    batch_tokens = token_dataset[:32][\"tokens\"]\n",
+    "    _, cache = model.run_with_cache(batch_tokens, prepend_bos=True)\n",
+    "\n",
+    "    # Use the SAE\n",
+    "    feature_acts = sae.encode(cache[sae.cfg.hook_name])\n",
+    "    sae_out = sae.decode(feature_acts)\n",
+    "\n",
+    "    # save some room\n",
+    "    del cache\n",
+    "\n",
+    "    # ignore the bos token, get the number of features that activated in each token, averaged accross batch and position\n",
+    "    l0 = (feature_acts[:, 1:] > 0).float().sum(-1).detach()\n",
+    "    print(\"average l0\", l0.mean().item())\n",
+    "    px.histogram(l0.flatten().cpu().numpy()).show()"
+   ],
+   "id": "a1f9a9f823253259",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "?\n",
+      "average l0 64.1279525756836\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.plotly.v1+json": {
+       "data": [
+        {
+         "alignmentgroup": "True",
+         "bingroup": "x",
+         "hovertemplate": "variable=0<br>value=%{x}<br>count=%{y}<extra></extra>",
+         "legendgroup": "0",
+         "marker": {
+          "color": "#636efa",
+          "pattern": {
+           "shape": ""
+          }
+         },
+         "name": "0",
+         "offsetgroup": "0",
+         "orientation": "v",
+         "showlegend": true,
+         "x": [
+          15.0,
+          25.0,
+          51.0,
+          84.0,
+          88.0,
+          100.0,
+          85.0,
+          60.0,
+          49.0,
+          47.0,
+          36.0,
+          37.0,
+          72.0,
+          88.0,
+          81.0,
+          74.0,
+          50.0,
+          66.0,
+          59.0,
+          38.0,
+          57.0,
+          67.0,
+          38.0,
+          68.0,
+          54.0,
+          68.0,
+          62.0,
+          74.0,
+          66.0,
+          53.0,
+          85.0,
+          76.0,
+          92.0,
+          59.0,
+          73.0,
+          52.0,
+          46.0,
+          51.0,
+          42.0,
+          81.0,
+          49.0,
+          42.0,
+          77.0,
+          90.0,
+          60.0,
+          93.0,
+          70.0,
+          77.0,
+          70.0,
+          59.0,
+          74.0,
+          66.0,
+          71.0,
+          51.0,
+          43.0,
+          44.0,
+          39.0,
+          22.0,
+          30.0,
+          44.0,
+          44.0,
+          34.0,
+          59.0,
+          36.0,
+          52.0,
+          60.0,
+          57.0,
+          45.0,
+          62.0,
+          55.0,
+          75.0,
+          43.0,
+          22.0,
+          37.0,
+          41.0,
+          40.0,
+          60.0,
+          50.0,
+          57.0,
+          74.0,
+          53.0,
+          84.0,
+          120.0,
+          78.0,
+          76.0,
+          67.0,
+          72.0,
+          68.0,
+          101.0,
+          78.0,
+          87.0,
+          99.0,
+          85.0,
+          47.0,
+          48.0,
+          30.0,
+          76.0,
+          65.0,
+          63.0,
+          49.0,
+          45.0,
+          70.0,
+          79.0,
+          76.0,
+          74.0,
+          71.0,
+          66.0,
+          69.0,
+          97.0,
+          81.0,
+          65.0,
+          69.0,
+          83.0,
+          84.0,
+          65.0,
+          58.0,
+          77.0,
+          63.0,
+          66.0,
+          64.0,
+          64.0,
+          72.0,
+          66.0,
+          90.0,
+          75.0,
+          59.0,
+          75.0,
+          27.0,
+          47.0,
+          45.0,
+          55.0,
+          54.0,
+          76.0,
+          66.0,
+          90.0,
+          98.0,
+          66.0,
+          77.0,
+          71.0,
+          79.0,
+          80.0,
+          60.0,
+          63.0,
+          91.0,
+          82.0,
+          65.0,
+          59.0,
+          70.0,
+          63.0,
+          73.0,
+          72.0,
+          63.0,
+          87.0,
+          81.0,
+          78.0,
+          86.0,
+          61.0,
+          59.0,
+          98.0,
+          84.0,
+          65.0,
+          63.0,
+          51.0,
+          63.0,
+          61.0,
+          72.0,
+          78.0,
+          85.0,
+          79.0,
+          75.0,
+          86.0,
+          51.0,
+          37.0,
+          48.0,
+          51.0,
+          49.0,
+          54.0,
+          58.0,
+          67.0,
+          41.0,
+          49.0,
+          68.0,
+          68.0,
+          88.0,
+          40.0,
+          42.0,
+          49.0,
+          90.0,
+          49.0,
+          65.0,
+          87.0,
+          77.0,
+          39.0,
+          75.0,
+          54.0,
+          70.0,
+          57.0,
+          43.0,
+          96.0,
+          51.0,
+          45.0,
+          61.0,
+          63.0,
+          61.0,
+          90.0,
+          52.0,
+          89.0,
+          60.0,
+          77.0,
+          62.0,
+          71.0,
+          62.0,
+          74.0,
+          105.0,
+          89.0,
+          118.0,
+          71.0,
+          67.0,
+          45.0,
+          53.0,
+          58.0,
+          82.0,
+          76.0,
+          45.0,
+          53.0,
+          43.0,
+          71.0,
+          86.0,
+          71.0,
+          51.0,
+          48.0,
+          51.0,
+          84.0,
+          79.0,
+          87.0,
+          78.0,
+          68.0,
+          94.0,
+          74.0,
+          64.0,
+          68.0,
+          38.0,
+          53.0,
+          57.0,
+          57.0,
+          78.0,
+          68.0,
+          39.0,
+          44.0,
+          49.0,
+          57.0,
+          65.0,
+          62.0,
+          60.0,
+          30.0,
+          49.0,
+          59.0,
+          66.0,
+          71.0,
+          55.0,
+          66.0,
+          66.0,
+          63.0,
+          52.0,
+          84.0,
+          76.0,
+          90.0,
+          73.0,
+          71.0,
+          85.0,
+          77.0,
+          82.0,
+          72.0,
+          68.0,
+          58.0,
+          46.0,
+          49.0,
+          57.0,
+          75.0,
+          46.0,
+          64.0,
+          53.0,
+          55.0,
+          67.0,
+          79.0,
+          88.0,
+          72.0,
+          58.0,
+          28.0,
+          39.0,
+          44.0,
+          47.0,
+          92.0,
+          98.0,
+          72.0,
+          83.0,
+          25.0,
+          37.0,
+          82.0,
+          75.0,
+          55.0,
+          69.0,
+          80.0,
+          82.0,
+          71.0,
+          64.0,
+          50.0,
+          96.0,
+          71.0,
+          71.0,
+          74.0,
+          75.0,
+          82.0,
+          86.0,
+          79.0,
+          85.0,
+          83.0,
+          72.0,
+          68.0,
+          55.0,
+          40.0,
+          49.0,
+          76.0,
+          82.0,
+          83.0,
+          78.0,
+          70.0,
+          108.0,
+          81.0,
+          54.0,
+          22.0,
+          40.0,
+          41.0,
+          59.0,
+          42.0,
+          48.0,
+          68.0,
+          70.0,
+          95.0,
+          120.0,
+          75.0,
+          52.0,
+          32.0,
+          33.0,
+          21.0,
+          69.0,
+          57.0,
+          52.0,
+          55.0,
+          48.0,
+          47.0,
+          91.0,
+          60.0,
+          68.0,
+          54.0,
+          62.0,
+          65.0,
+          75.0,
+          74.0,
+          73.0,
+          71.0,
+          87.0,
+          61.0,
+          57.0,
+          75.0,
+          83.0,
+          73.0,
+          104.0,
+          86.0,
+          112.0,
+          82.0,
+          74.0,
+          72.0,
+          53.0,
+          54.0,
+          27.0,
+          35.0,
+          61.0,
+          65.0,
+          70.0,
+          70.0,
+          6.0,
+          26.0,
+          21.0,
+          42.0,
+          71.0,
+          87.0,
+          32.0,
+          45.0,
+          88.0,
+          65.0,
+          74.0,
+          62.0,
+          68.0,
+          65.0,
+          55.0,
+          40.0,
+          38.0,
+          28.0,
+          34.0,
+          34.0,
+          42.0,
+          47.0,
+          78.0,
+          47.0,
+          72.0,
+          78.0,
+          61.0,
+          79.0,
+          106.0,
+          75.0,
+          95.0,
+          68.0,
+          70.0,
+          49.0,
+          54.0,
+          69.0,
+          73.0,
+          85.0,
+          69.0,
+          71.0,
+          56.0,
+          64.0,
+          77.0,
+          84.0,
+          79.0,
+          90.0,
+          86.0,
+          79.0,
+          34.0,
+          27.0,
+          29.0,
+          37.0,
+          46.0,
+          55.0,
+          53.0,
+          48.0,
+          48.0,
+          58.0,
+          58.0,
+          52.0,
+          61.0,
+          58.0,
+          42.0,
+          75.0,
+          83.0,
+          60.0,
+          63.0,
+          39.0,
+          33.0,
+          52.0,
+          46.0,
+          55.0,
+          29.0,
+          34.0,
+          51.0,
+          54.0,
+          64.0,
+          90.0,
+          63.0,
+          59.0,
+          91.0,
+          62.0,
+          77.0,
+          87.0,
+          74.0,
+          39.0,
+          44.0,
+          32.0,
+          84.0,
+          53.0,
+          32.0,
+          41.0,
+          46.0,
+          45.0,
+          48.0,
+          68.0,
+          78.0,
+          41.0,
+          45.0,
+          54.0,
+          72.0,
+          61.0,
+          70.0,
+          62.0,
+          54.0,
+          71.0,
+          80.0,
+          92.0,
+          89.0,
+          73.0,
+          99.0,
+          85.0,
+          83.0,
+          92.0,
+          79.0,
+          67.0,
+          68.0,
+          78.0,
+          90.0,
+          72.0,
+          80.0,
+          95.0,
+          78.0,
+          75.0,
+          48.0,
+          47.0,
+          61.0
+         ],
+         "xaxis": "x",
+         "yaxis": "y",
+         "type": "histogram"
+        }
+       ],
+       "layout": {
+        "template": {
+         "data": {
+          "histogram2dcontour": [
+           {
+            "type": "histogram2dcontour",
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "colorscale": [
+             [
+              0.0,
+              "#0d0887"
+             ],
+             [
+              0.1111111111111111,
+              "#46039f"
+             ],
+             [
+              0.2222222222222222,
+              "#7201a8"
+             ],
+             [
+              0.3333333333333333,
+              "#9c179e"
+             ],
+             [
+              0.4444444444444444,
+              "#bd3786"
+             ],
+             [
+              0.5555555555555556,
+              "#d8576b"
+             ],
+             [
+              0.6666666666666666,
+              "#ed7953"
+             ],
+             [
+              0.7777777777777778,
+              "#fb9f3a"
+             ],
+             [
+              0.8888888888888888,
+              "#fdca26"
+             ],
+             [
+              1.0,
+              "#f0f921"
+             ]
+            ]
+           }
+          ],
+          "choropleth": [
+           {
+            "type": "choropleth",
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            }
+           }
+          ],
+          "histogram2d": [
+           {
+            "type": "histogram2d",
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "colorscale": [
+             [
+              0.0,
+              "#0d0887"
+             ],
+             [
+              0.1111111111111111,
+              "#46039f"
+             ],
+             [
+              0.2222222222222222,
+              "#7201a8"
+             ],
+             [
+              0.3333333333333333,
+              "#9c179e"
+             ],
+             [
+              0.4444444444444444,
+              "#bd3786"
+             ],
+             [
+              0.5555555555555556,
+              "#d8576b"
+             ],
+             [
+              0.6666666666666666,
+              "#ed7953"
+             ],
+             [
+              0.7777777777777778,
+              "#fb9f3a"
+             ],
+             [
+              0.8888888888888888,
+              "#fdca26"
+             ],
+             [
+              1.0,
+              "#f0f921"
+             ]
+            ]
+           }
+          ],
+          "heatmap": [
+           {
+            "type": "heatmap",
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "colorscale": [
+             [
+              0.0,
+              "#0d0887"
+             ],
+             [
+              0.1111111111111111,
+              "#46039f"
+             ],
+             [
+              0.2222222222222222,
+              "#7201a8"
+             ],
+             [
+              0.3333333333333333,
+              "#9c179e"
+             ],
+             [
+              0.4444444444444444,
+              "#bd3786"
+             ],
+             [
+              0.5555555555555556,
+              "#d8576b"
+             ],
+             [
+              0.6666666666666666,
+              "#ed7953"
+             ],
+             [
+              0.7777777777777778,
+              "#fb9f3a"
+             ],
+             [
+              0.8888888888888888,
+              "#fdca26"
+             ],
+             [
+              1.0,
+              "#f0f921"
+             ]
+            ]
+           }
+          ],
+          "heatmapgl": [
+           {
+            "type": "heatmapgl",
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "colorscale": [
+             [
+              0.0,
+              "#0d0887"
+             ],
+             [
+              0.1111111111111111,
+              "#46039f"
+             ],
+             [
+              0.2222222222222222,
+              "#7201a8"
+             ],
+             [
+              0.3333333333333333,
+              "#9c179e"
+             ],
+             [
+              0.4444444444444444,
+              "#bd3786"
+             ],
+             [
+              0.5555555555555556,
+              "#d8576b"
+             ],
+             [
+              0.6666666666666666,
+              "#ed7953"
+             ],
+             [
+              0.7777777777777778,
+              "#fb9f3a"
+             ],
+             [
+              0.8888888888888888,
+              "#fdca26"
+             ],
+             [
+              1.0,
+              "#f0f921"
+             ]
+            ]
+           }
+          ],
+          "contourcarpet": [
+           {
+            "type": "contourcarpet",
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            }
+           }
+          ],
+          "contour": [
+           {
+            "type": "contour",
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "colorscale": [
+             [
+              0.0,
+              "#0d0887"
+             ],
+             [
+              0.1111111111111111,
+              "#46039f"
+             ],
+             [
+              0.2222222222222222,
+              "#7201a8"
+             ],
+             [
+              0.3333333333333333,
+              "#9c179e"
+             ],
+             [
+              0.4444444444444444,
+              "#bd3786"
+             ],
+             [
+              0.5555555555555556,
+              "#d8576b"
+             ],
+             [
+              0.6666666666666666,
+              "#ed7953"
+             ],
+             [
+              0.7777777777777778,
+              "#fb9f3a"
+             ],
+             [
+              0.8888888888888888,
+              "#fdca26"
+             ],
+             [
+              1.0,
+              "#f0f921"
+             ]
+            ]
+           }
+          ],
+          "surface": [
+           {
+            "type": "surface",
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            },
+            "colorscale": [
+             [
+              0.0,
+              "#0d0887"
+             ],
+             [
+              0.1111111111111111,
+              "#46039f"
+             ],
+             [
+              0.2222222222222222,
+              "#7201a8"
+             ],
+             [
+              0.3333333333333333,
+              "#9c179e"
+             ],
+             [
+              0.4444444444444444,
+              "#bd3786"
+             ],
+             [
+              0.5555555555555556,
+              "#d8576b"
+             ],
+             [
+              0.6666666666666666,
+              "#ed7953"
+             ],
+             [
+              0.7777777777777778,
+              "#fb9f3a"
+             ],
+             [
+              0.8888888888888888,
+              "#fdca26"
+             ],
+             [
+              1.0,
+              "#f0f921"
+             ]
+            ]
+           }
+          ],
+          "mesh3d": [
+           {
+            "type": "mesh3d",
+            "colorbar": {
+             "outlinewidth": 0,
+             "ticks": ""
+            }
+           }
+          ],
+          "scatter": [
+           {
+            "marker": {
+             "line": {
+              "color": "#283442"
+             }
+            },
+            "type": "scatter"
+           }
+          ],
+          "parcoords": [
+           {
+            "type": "parcoords",
+            "line": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            }
+           }
+          ],
+          "scatterpolargl": [
+           {
+            "type": "scatterpolargl",
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            }
+           }
+          ],
+          "bar": [
+           {
+            "error_x": {
+             "color": "#f2f5fa"
+            },
+            "error_y": {
+             "color": "#f2f5fa"
+            },
+            "marker": {
+             "line": {
+              "color": "rgb(17,17,17)",
+              "width": 0.5
+             },
+             "pattern": {
+              "fillmode": "overlay",
+              "size": 10,
+              "solidity": 0.2
+             }
+            },
+            "type": "bar"
+           }
+          ],
+          "scattergeo": [
+           {
+            "type": "scattergeo",
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            }
+           }
+          ],
+          "scatterpolar": [
+           {
+            "type": "scatterpolar",
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            }
+           }
+          ],
+          "histogram": [
+           {
+            "marker": {
+             "pattern": {
+              "fillmode": "overlay",
+              "size": 10,
+              "solidity": 0.2
+             }
+            },
+            "type": "histogram"
+           }
+          ],
+          "scattergl": [
+           {
+            "marker": {
+             "line": {
+              "color": "#283442"
+             }
+            },
+            "type": "scattergl"
+           }
+          ],
+          "scatter3d": [
+           {
+            "type": "scatter3d",
+            "line": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            },
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            }
+           }
+          ],
+          "scattermapbox": [
+           {
+            "type": "scattermapbox",
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            }
+           }
+          ],
+          "scatterternary": [
+           {
+            "type": "scatterternary",
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            }
+           }
+          ],
+          "scattercarpet": [
+           {
+            "type": "scattercarpet",
+            "marker": {
+             "colorbar": {
+              "outlinewidth": 0,
+              "ticks": ""
+             }
+            }
+           }
+          ],
+          "carpet": [
+           {
+            "aaxis": {
+             "endlinecolor": "#A2B1C6",
+             "gridcolor": "#506784",
+             "linecolor": "#506784",
+             "minorgridcolor": "#506784",
+             "startlinecolor": "#A2B1C6"
+            },
+            "baxis": {
+             "endlinecolor": "#A2B1C6",
+             "gridcolor": "#506784",
+             "linecolor": "#506784",
+             "minorgridcolor": "#506784",
+             "startlinecolor": "#A2B1C6"
+            },
+            "type": "carpet"
+           }
+          ],
+          "table": [
+           {
+            "cells": {
+             "fill": {
+              "color": "#506784"
+             },
+             "line": {
+              "color": "rgb(17,17,17)"
+             }
+            },
+            "header": {
+             "fill": {
+              "color": "#2a3f5f"
+             },
+             "line": {
+              "color": "rgb(17,17,17)"
+             }
+            },
+            "type": "table"
+           }
+          ],
+          "barpolar": [
+           {
+            "marker": {
+             "line": {
+              "color": "rgb(17,17,17)",
+              "width": 0.5
+             },
+             "pattern": {
+              "fillmode": "overlay",
+              "size": 10,
+              "solidity": 0.2
+             }
+            },
+            "type": "barpolar"
+           }
+          ],
+          "pie": [
+           {
+            "automargin": true,
+            "type": "pie"
+           }
+          ]
+         },
+         "layout": {
+          "autotypenumbers": "strict",
+          "colorway": [
+           "#636efa",
+           "#EF553B",
+           "#00cc96",
+           "#ab63fa",
+           "#FFA15A",
+           "#19d3f3",
+           "#FF6692",
+           "#B6E880",
+           "#FF97FF",
+           "#FECB52"
+          ],
+          "font": {
+           "color": "#f2f5fa"
+          },
+          "hovermode": "closest",
+          "hoverlabel": {
+           "align": "left"
+          },
+          "paper_bgcolor": "rgb(17,17,17)",
+          "plot_bgcolor": "rgb(17,17,17)",
+          "polar": {
+           "bgcolor": "rgb(17,17,17)",
+           "angularaxis": {
+            "gridcolor": "#506784",
+            "linecolor": "#506784",
+            "ticks": ""
+           },
+           "radialaxis": {
+            "gridcolor": "#506784",
+            "linecolor": "#506784",
+            "ticks": ""
+           }
+          },
+          "ternary": {
+           "bgcolor": "rgb(17,17,17)",
+           "aaxis": {
+            "gridcolor": "#506784",
+            "linecolor": "#506784",
+            "ticks": ""
+           },
+           "baxis": {
+            "gridcolor": "#506784",
+            "linecolor": "#506784",
+            "ticks": ""
+           },
+           "caxis": {
+            "gridcolor": "#506784",
+            "linecolor": "#506784",
+            "ticks": ""
+           }
+          },
+          "coloraxis": {
+           "colorbar": {
+            "outlinewidth": 0,
+            "ticks": ""
+           }
+          },
+          "colorscale": {
+           "sequential": [
+            [
+             0.0,
+             "#0d0887"
+            ],
+            [
+             0.1111111111111111,
+             "#46039f"
+            ],
+            [
+             0.2222222222222222,
+             "#7201a8"
+            ],
+            [
+             0.3333333333333333,
+             "#9c179e"
+            ],
+            [
+             0.4444444444444444,
+             "#bd3786"
+            ],
+            [
+             0.5555555555555556,
+             "#d8576b"
+            ],
+            [
+             0.6666666666666666,
+             "#ed7953"
+            ],
+            [
+             0.7777777777777778,
+             "#fb9f3a"
+            ],
+            [
+             0.8888888888888888,
+             "#fdca26"
+            ],
+            [
+             1.0,
+             "#f0f921"
+            ]
+           ],
+           "sequentialminus": [
+            [
+             0.0,
+             "#0d0887"
+            ],
+            [
+             0.1111111111111111,
+             "#46039f"
+            ],
+            [
+             0.2222222222222222,
+             "#7201a8"
+            ],
+            [
+             0.3333333333333333,
+             "#9c179e"
+            ],
+            [
+             0.4444444444444444,
+             "#bd3786"
+            ],
+            [
+             0.5555555555555556,
+             "#d8576b"
+            ],
+            [
+             0.6666666666666666,
+             "#ed7953"
+            ],
+            [
+             0.7777777777777778,
+             "#fb9f3a"
+            ],
+            [
+             0.8888888888888888,
+             "#fdca26"
+            ],
+            [
+             1.0,
+             "#f0f921"
+            ]
+           ],
+           "diverging": [
+            [
+             0,
+             "#8e0152"
+            ],
+            [
+             0.1,
+             "#c51b7d"
+            ],
+            [
+             0.2,
+             "#de77ae"
+            ],
+            [
+             0.3,
+             "#f1b6da"
+            ],
+            [
+             0.4,
+             "#fde0ef"
+            ],
+            [
+             0.5,
+             "#f7f7f7"
+            ],
+            [
+             0.6,
+             "#e6f5d0"
+            ],
+            [
+             0.7,
+             "#b8e186"
+            ],
+            [
+             0.8,
+             "#7fbc41"
+            ],
+            [
+             0.9,
+             "#4d9221"
+            ],
+            [
+             1,
+             "#276419"
+            ]
+           ]
+          },
+          "xaxis": {
+           "gridcolor": "#283442",
+           "linecolor": "#506784",
+           "ticks": "",
+           "title": {
+            "standoff": 15
+           },
+           "zerolinecolor": "#283442",
+           "automargin": true,
+           "zerolinewidth": 2
+          },
+          "yaxis": {
+           "gridcolor": "#283442",
+           "linecolor": "#506784",
+           "ticks": "",
+           "title": {
+            "standoff": 15
+           },
+           "zerolinecolor": "#283442",
+           "automargin": true,
+           "zerolinewidth": 2
+          },
+          "scene": {
+           "xaxis": {
+            "backgroundcolor": "rgb(17,17,17)",
+            "gridcolor": "#506784",
+            "linecolor": "#506784",
+            "showbackground": true,
+            "ticks": "",
+            "zerolinecolor": "#C8D4E3",
+            "gridwidth": 2
+           },
+           "yaxis": {
+            "backgroundcolor": "rgb(17,17,17)",
+            "gridcolor": "#506784",
+            "linecolor": "#506784",
+            "showbackground": true,
+            "ticks": "",
+            "zerolinecolor": "#C8D4E3",
+            "gridwidth": 2
+           },
+           "zaxis": {
+            "backgroundcolor": "rgb(17,17,17)",
+            "gridcolor": "#506784",
+            "linecolor": "#506784",
+            "showbackground": true,
+            "ticks": "",
+            "zerolinecolor": "#C8D4E3",
+            "gridwidth": 2
+           }
+          },
+          "shapedefaults": {
+           "line": {
+            "color": "#f2f5fa"
+           }
+          },
+          "annotationdefaults": {
+           "arrowcolor": "#f2f5fa",
+           "arrowhead": 0,
+           "arrowwidth": 1
+          },
+          "geo": {
+           "bgcolor": "rgb(17,17,17)",
+           "landcolor": "rgb(17,17,17)",
+           "subunitcolor": "#506784",
+           "showland": true,
+           "showlakes": true,
+           "lakecolor": "rgb(17,17,17)"
+          },
+          "title": {
+           "x": 0.05
+          },
+          "updatemenudefaults": {
+           "bgcolor": "#506784",
+           "borderwidth": 0
+          },
+          "sliderdefaults": {
+           "bgcolor": "#C8D4E3",
+           "borderwidth": 1,
+           "bordercolor": "rgb(17,17,17)",
+           "tickwidth": 0
+          },
+          "mapbox": {
+           "style": "dark"
+          }
+         }
+        },
+        "xaxis": {
+         "anchor": "y",
+         "domain": [
+          0.0,
+          1.0
+         ],
+         "title": {
+          "text": "value"
+         }
+        },
+        "yaxis": {
+         "anchor": "x",
+         "domain": [
+          0.0,
+          1.0
+         ],
+         "title": {
+          "text": "count"
+         }
+        },
+        "legend": {
+         "title": {
+          "text": "variable"
+         },
+         "tracegroupgap": 0
+        },
+        "margin": {
+         "t": 60
+        },
+        "barmode": "relative"
+       },
+       "config": {
+        "plotlyServerURL": "https://plot.ly"
+       }
+      },
+      "text/html": [
+       "<div>                            <div id=\"123c5b41-465a-466f-8bd4-1be7a1927f18\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"123c5b41-465a-466f-8bd4-1be7a1927f18\")) {                    Plotly.newPlot(                        \"123c5b41-465a-466f-8bd4-1be7a1927f18\",                        [{\"alignmentgroup\":\"True\",\"bingroup\":\"x\",\"hovertemplate\":\"variable=0\\u003cbr\\u003evalue=%{x}\\u003cbr\\u003ecount=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"marker\":{\"color\":\"#636efa\",\"pattern\":{\"shape\":\"\"}},\"name\":\"0\",\"offsetgroup\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[15.0,25.0,51.0,84.0,88.0,100.0,85.0,60.0,49.0,47.0,36.0,37.0,72.0,88.0,81.0,74.0,50.0,66.0,59.0,38.0,57.0,67.0,38.0,68.0,54.0,68.0,62.0,74.0,66.0,53.0,85.0,76.0,92.0,59.0,73.0,52.0,46.0,51.0,42.0,81.0,49.0,42.0,77.0,90.0,60.0,93.0,70.0,77.0,70.0,59.0,74.0,66.0,71.0,51.0,43.0,44.0,39.0,22.0,30.0,44.0,44.0,34.0,59.0,36.0,52.0,60.0,57.0,45.0,62.0,55.0,75.0,43.0,22.0,37.0,41.0,40.0,60.0,50.0,57.0,74.0,53.0,84.0,120.0,78.0,76.0,67.0,72.0,68.0,101.0,78.0,87.0,99.0,85.0,47.0,48.0,30.0,76.0,65.0,63.0,49.0,45.0,70.0,79.0,76.0,74.0,71.0,66.0,69.0,97.0,81.0,65.0,69.0,83.0,84.0,65.0,58.0,77.0,63.0,66.0,64.0,64.0,72.0,66.0,90.0,75.0,59.0,75.0,27.0,47.0,45.0,55.0,54.0,76.0,66.0,90.0,98.0,66.0,77.0,71.0,79.0,80.0,60.0,63.0,91.0,82.0,65.0,59.0,70.0,63.0,73.0,72.0,63.0,87.0,81.0,78.0,86.0,61.0,59.0,98.0,84.0,65.0,63.0,51.0,63.0,61.0,72.0,78.0,85.0,79.0,75.0,86.0,51.0,37.0,48.0,51.0,49.0,54.0,58.0,67.0,41.0,49.0,68.0,68.0,88.0,40.0,42.0,49.0,90.0,49.0,65.0,87.0,77.0,39.0,75.0,54.0,70.0,57.0,43.0,96.0,51.0,45.0,61.0,63.0,61.0,90.0,52.0,89.0,60.0,77.0,62.0,71.0,62.0,74.0,105.0,89.0,118.0,71.0,67.0,45.0,53.0,58.0,82.0,76.0,45.0,53.0,43.0,71.0,86.0,71.0,51.0,48.0,51.0,84.0,79.0,87.0,78.0,68.0,94.0,74.0,64.0,68.0,38.0,53.0,57.0,57.0,78.0,68.0,39.0,44.0,49.0,57.0,65.0,62.0,60.0,30.0,49.0,59.0,66.0,71.0,55.0,66.0,66.0,63.0,52.0,84.0,76.0,90.0,73.0,71.0,85.0,77.0,82.0,72.0,68.0,58.0,46.0,49.0,57.0,75.0,46.0,64.0,53.0,55.0,67.0,79.0,88.0,72.0,58.0,28.0,39.0,44.0,47.0,92.0,98.0,72.0,83.0,25.0,37.0,82.0,75.0,55.0,69.0,80.0,82.0,71.0,64.0,50.0,96.0,71.0,71.0,74.0,75.0,82.0,86.0,79.0,85.0,83.0,72.0,68.0,55.0,40.0,49.0,76.0,82.0,83.0,78.0,70.0,108.0,81.0,54.0,22.0,40.0,41.0,59.0,42.0,48.0,68.0,70.0,95.0,120.0,75.0,52.0,32.0,33.0,21.0,69.0,57.0,52.0,55.0,48.0,47.0,91.0,60.0,68.0,54.0,62.0,65.0,75.0,74.0,73.0,71.0,87.0,61.0,57.0,75.0,83.0,73.0,104.0,86.0,112.0,82.0,74.0,72.0,53.0,54.0,27.0,35.0,61.0,65.0,70.0,70.0,6.0,26.0,21.0,42.0,71.0,87.0,32.0,45.0,88.0,65.0,74.0,62.0,68.0,65.0,55.0,40.0,38.0,28.0,34.0,34.0,42.0,47.0,78.0,47.0,72.0,78.0,61.0,79.0,106.0,75.0,95.0,68.0,70.0,49.0,54.0,69.0,73.0,85.0,69.0,71.0,56.0,64.0,77.0,84.0,79.0,90.0,86.0,79.0,34.0,27.0,29.0,37.0,46.0,55.0,53.0,48.0,48.0,58.0,58.0,52.0,61.0,58.0,42.0,75.0,83.0,60.0,63.0,39.0,33.0,52.0,46.0,55.0,29.0,34.0,51.0,54.0,64.0,90.0,63.0,59.0,91.0,62.0,77.0,87.0,74.0,39.0,44.0,32.0,84.0,53.0,32.0,41.0,46.0,45.0,48.0,68.0,78.0,41.0,45.0,54.0,72.0,61.0,70.0,62.0,54.0,71.0,80.0,92.0,89.0,73.0,99.0,85.0,83.0,92.0,79.0,67.0,68.0,78.0,90.0,72.0,80.0,95.0,78.0,75.0,48.0,47.0,61.0],\"xaxis\":\"x\",\"yaxis\":\"y\",\"type\":\"histogram\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"marker\":{\"line\":{\"color\":\"#283442\"}},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#f2f5fa\"},\"error_y\":{\"color\":\"#f2f5fa\"},\"marker\":{\"line\":{\"color\":\"rgb(17,17,17)\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"marker\":{\"line\":{\"color\":\"#283442\"}},\"type\":\"scattergl\"}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#A2B1C6\",\"gridcolor\":\"#506784\",\"linecolor\":\"#506784\",\"minorgridcolor\":\"#506784\",\"startlinecolor\":\"#A2B1C6\"},\"baxis\":{\"endlinecolor\":\"#A2B1C6\",\"gridcolor\":\"#506784\",\"linecolor\":\"#506784\",\"minorgridcolor\":\"#506784\",\"startlinecolor\":\"#A2B1C6\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#506784\"},\"line\":{\"color\":\"rgb(17,17,17)\"}},\"header\":{\"fill\":{\"color\":\"#2a3f5f\"},\"line\":{\"color\":\"rgb(17,17,17)\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"rgb(17,17,17)\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#f2f5fa\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"rgb(17,17,17)\",\"plot_bgcolor\":\"rgb(17,17,17)\",\"polar\":{\"bgcolor\":\"rgb(17,17,17)\",\"angularaxis\":{\"gridcolor\":\"#506784\",\"linecolor\":\"#506784\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"#506784\",\"linecolor\":\"#506784\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"rgb(17,17,17)\",\"aaxis\":{\"gridcolor\":\"#506784\",\"linecolor\":\"#506784\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"#506784\",\"linecolor\":\"#506784\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"#506784\",\"linecolor\":\"#506784\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"#283442\",\"linecolor\":\"#506784\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"#283442\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"#283442\",\"linecolor\":\"#506784\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"#283442\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"rgb(17,17,17)\",\"gridcolor\":\"#506784\",\"linecolor\":\"#506784\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"#C8D4E3\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"rgb(17,17,17)\",\"gridcolor\":\"#506784\",\"linecolor\":\"#506784\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"#C8D4E3\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"rgb(17,17,17)\",\"gridcolor\":\"#506784\",\"linecolor\":\"#506784\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"#C8D4E3\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#f2f5fa\"}},\"annotationdefaults\":{\"arrowcolor\":\"#f2f5fa\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"rgb(17,17,17)\",\"landcolor\":\"rgb(17,17,17)\",\"subunitcolor\":\"#506784\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"rgb(17,17,17)\"},\"title\":{\"x\":0.05},\"updatemenudefaults\":{\"bgcolor\":\"#506784\",\"borderwidth\":0},\"sliderdefaults\":{\"bgcolor\":\"#C8D4E3\",\"borderwidth\":1,\"bordercolor\":\"rgb(17,17,17)\",\"tickwidth\":0},\"mapbox\":{\"style\":\"dark\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"count\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"margin\":{\"t\":60},\"barmode\":\"relative\"},                        {\"responsive\": true}                    ).then(function(){\n",
+       "                            \n",
+       "var gd = document.getElementById('123c5b41-465a-466f-8bd4-1be7a1927f18');\n",
+       "var x = new MutationObserver(function (mutations, observer) {{\n",
+       "        var display = window.getComputedStyle(gd).display;\n",
+       "        if (!display || display === 'none') {{\n",
+       "            console.log([gd, 'removed!']);\n",
+       "            Plotly.purge(gd);\n",
+       "            observer.disconnect();\n",
+       "        }}\n",
+       "}});\n",
+       "\n",
+       "// Listen for the removal of the full notebook cells\n",
+       "var notebookContainer = gd.closest('#notebook-container');\n",
+       "if (notebookContainer) {{\n",
+       "    x.observe(notebookContainer, {childList: true});\n",
+       "}}\n",
+       "\n",
+       "// Listen for the clearing of the current output cell\n",
+       "var outputEl = gd.closest('.output');\n",
+       "if (outputEl) {{\n",
+       "    x.observe(outputEl, {childList: true});\n",
+       "}}\n",
+       "\n",
+       "                        })                };                });            </script>        </div>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "execution_count": 6
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T01:39:16.853090Z",
+     "start_time": "2024-11-17T01:39:16.170397Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "from transformer_lens import utils\n",
+    "from functools import partial\n",
+    "\n",
+    "\n",
+    "# next we want to do a reconstruction test.\n",
+    "def reconstr_hook(activation, hook, sae_out):\n",
+    "    return sae_out\n",
+    "\n",
+    "\n",
+    "def zero_abl_hook(activation, hook):\n",
+    "    return torch.zeros_like(activation)\n",
+    "\n",
+    "\n",
+    "print(\"Orig\", model(batch_tokens, return_type=\"loss\").item())\n",
+    "print(\n",
+    "    \"reconstr\",\n",
+    "    model.run_with_hooks(\n",
+    "        batch_tokens,\n",
+    "        fwd_hooks=[\n",
+    "            (\n",
+    "                sae.cfg.hook_name,\n",
+    "                partial(reconstr_hook, sae_out=sae_out),\n",
+    "            )\n",
+    "        ],\n",
+    "        return_type=\"loss\",\n",
+    "    ).item(),\n",
+    ")\n",
+    "print(\n",
+    "    \"Zero\",\n",
+    "    model.run_with_hooks(\n",
+    "        batch_tokens,\n",
+    "        return_type=\"loss\",\n",
+    "        fwd_hooks=[(sae.cfg.hook_name, zero_abl_hook)],\n",
+    "    ).item(),\n",
+    ")"
+   ],
+   "id": "ddabe8530685c45",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Orig 3.5622000694274902\n",
+      "reconstr 3.764155387878418\n",
+      "Zero 11.146590232849121\n"
+     ]
+    }
+   ],
+   "execution_count": 5
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T01:39:48.048784Z",
+     "start_time": "2024-11-17T01:39:48.033476Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "cfg_dict",
+   "id": "f08540e9e717e9fe",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'model_name': 'gpt2-small',\n",
+       " 'hook_point': 'blocks.8.hook_resid_pre',\n",
+       " 'hook_point_layer': 8,\n",
+       " 'hook_point_head_index': None,\n",
+       " 'dataset_path': 'Skylion007/openwebtext',\n",
+       " 'is_dataset_tokenized': False,\n",
+       " 'context_size': 128,\n",
+       " 'use_cached_activations': False,\n",
+       " 'cached_activations_path': 'activations/Skylion007_openwebtext/gpt2-small/blocks.8.hook_resid_pre',\n",
+       " 'd_in': 768,\n",
+       " 'n_batches_in_buffer': 128,\n",
+       " 'total_training_tokens': 300000000,\n",
+       " 'store_batch_size': 32,\n",
+       " 'device': 'cuda',\n",
+       " 'seed': 42,\n",
+       " 'dtype': 'torch.float32',\n",
+       " 'b_dec_init_method': 'geometric_median',\n",
+       " 'expansion_factor': 32,\n",
+       " 'from_pretrained_path': None,\n",
+       " 'l1_coefficient': 8e-05,\n",
+       " 'lr': 0.0004,\n",
+       " 'lr_scheduler_name': None,\n",
+       " 'lr_warm_up_steps': 5000,\n",
+       " 'train_batch_size': 4096,\n",
+       " 'use_ghost_grads': False,\n",
+       " 'feature_sampling_window': 1000,\n",
+       " 'feature_sampling_method': None,\n",
+       " 'resample_batches': 1028,\n",
+       " 'feature_reinit_scale': 0.2,\n",
+       " 'dead_feature_window': 5000,\n",
+       " 'dead_feature_estimation_method': 'no_fire',\n",
+       " 'dead_feature_threshold': 1e-08,\n",
+       " 'log_to_wandb': True,\n",
+       " 'wandb_project': 'mats_sae_training_gpt2_small_resid_pre_5',\n",
+       " 'wandb_entity': None,\n",
+       " 'wandb_log_frequency': 100,\n",
+       " 'n_checkpoints': 10,\n",
+       " 'checkpoint_path': 'checkpoints/ut7lhl4q',\n",
+       " 'd_sae': 24576,\n",
+       " 'tokens_per_buffer': 67108864,\n",
+       " 'run_name': '24576-L1-8e-05-LR-0.0004-Tokens-3.000e+08',\n",
+       " 'model_from_pretrained_kwargs': {'center_writing_weights': True},\n",
+       " 'neuronpedia_id': 'gpt2-small/8-res-jb',\n",
+       " 'prepend_bos': True,\n",
+       " 'dataset_trust_remote_code': True,\n",
+       " 'apply_b_dec_to_input': True,\n",
+       " 'finetuning_scaling_factor': False,\n",
+       " 'sae_lens_training_version': None,\n",
+       " 'activation_fn_str': 'relu',\n",
+       " 'architecture': 'standard',\n",
+       " 'normalize_activations': 'none'}"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 7
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-11-17T01:43:04.413424Z",
+     "start_time": "2024-11-17T01:43:04.407561Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "sae.W_dec.shape",
+   "id": "5e92bb48ae9ab956",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "torch.Size([24576, 768])"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 13
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "ab4398bacf9ee3bc"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

nnsight_gemma_steering_file.py

@@ -0,0 +1,99 @@
+from functools import partial
+from contextlib import contextmanager
+
+from nnsight import LanguageModel
+import torch
+#from transformer_lens import HookedTransformer, utils
+
+from sae_lens import SAE
+
+device = "cuda"
+
+sae_20, _, _ = SAE.from_pretrained(
+    release = "gemma-scope-2b-pt-res-canonical",
+    sae_id = "layer_20/width_16k/canonical",
+    device=device
+)
+sae_10, _, _ = SAE.from_pretrained(
+    release = "gemma-scope-2b-pt-res-canonical",
+    sae_id = "layer_10/width_16k/canonical",
+    device=device
+)
+
+sae_4, _, _ = SAE.from_pretrained(
+    release = "gemma-scope-2b-pt-res-canonical",
+    sae_id = "layer_4/width_16k/canonical",
+    device=device
+)
+sae_25, _, _ = SAE.from_pretrained(
+    release = "gemma-scope-2b-pt-res-canonical",
+    sae_id = "layer_25/width_16k/canonical",
+    device=device
+)
+feature_dict = {
+    "dog": {
+        "sae": sae_20,
+        "index": 12082
+    },
+    "harry potter4": {
+        "sae": sae_4,
+        "index": 12445
+    },
+    "harry potter10": {
+        "sae": sae_10,
+        "index": 6520
+    },
+    "kindness": {
+        "sae": sae_25,
+        "index": 10092
+    },
+    "yelling": {
+        "sae": sae_20,
+        "index": 11859
+    }
+}
+llm = LanguageModel(
+    "google/gemma-2-2b-it",
+    # dtype=torch.bfloat16,
+    # default_padding_side="left",
+    device_map="cuda:0",
+)
+# "meta-llama/Llama-3.2-1B-Instruct",#
+
+batched_chat = [
+    [
+        {"role": "user",
+         "content": "What book is Hermione Granger from?"}
+    ]
+]
+
+tokens = llm.tokenizer.apply_chat_template(batched_chat,
+    padding=True,
+    tokenize=True,
+    return_tensors="pt",
+    add_generation_prompt=True
+)
+
+
+feature = feature_dict["harry potter4"]
+strength = -5
+steering_vector = feature["sae"].W_dec[feature["index"]] * strength
+
+with llm.generate(tokens, temperature=1, max_new_tokens=128) as tracer:
+    for i in range(len(llm.model.layers)):
+        module_name = "post_attention_layernorm"
+        module = getattr(llm.model.layers[i], module_name)
+
+        resid_pre_before = module.output.clone().save()
+        module.output[:] = resid_pre_before + steering_vector
+
+        resid_pre_after = module.output.save()
+
+        # module.next()
+
+    output = llm.generator.output.save()
+
+# print("output tensors:", output)
+print("output string:", llm.tokenizer.batch_decode(output.tolist(), skip_special_tokens=False)[0])
+# print("Before:", resid_pre_before)
+# print("After:", resid_pre_after)

tlens_gemma_steering.py

@@ -0,0 +1,116 @@
+import os
+import torch
+from prometheus_client.decorator import contextmanager
+from tqdm import tqdm
+import plotly.express as px
+from datasets import load_dataset
+from transformer_lens import HookedTransformer, utils
+from functools import partial
+from sae_lens import SAE
+from contextlib import contextmanager
+device = "cuda"
+
+from sae_lens import SAE  # pip install sae-lens
+
+sae, cfg_dict, sparsity = SAE.from_pretrained(
+    release = "gemma-scope-2b-pt-res-canonical",
+    sae_id = "layer_20/width_16k/canonical",
+    device=device
+)
+
+sae_10, _, _ = SAE.from_pretrained(
+    release = "gemma-scope-2b-pt-res-canonical",
+    sae_id = "layer_10/width_16k/canonical",
+    device=device
+)
+
+sae_4, _, _ = SAE.from_pretrained(
+    release = "gemma-scope-2b-pt-res-canonical",
+    sae_id = "layer_4/width_16k/canonical",
+    device=device
+)
+
+model = HookedTransformer.from_pretrained_no_processing(
+    model_name="google/gemma-2-2b-it",
+    device=device,
+    dtype=torch.bfloat16,
+    default_padding_side="left"
+)
+layer = 20
+sae.eval()
+
+feature_dict = {
+    "dog": {
+        "sae": sae,
+        "index": 12082
+    },
+    "harry potter4": {
+        "sae": sae_4,
+        "index": 12445
+    },
+    "harry potter10": {
+        "sae": sae_10,
+        "index": 6520
+    }
+}
+
+def sae_hook(activation, hook, subject, strength):
+    feature = feature_dict[subject]
+    steering_vector = feature["sae"].W_dec[feature["index"]] * strength
+    return activation + steering_vector
+
+
+@contextmanager
+def steering(subject, strength):
+    layers = list(range(model.cfg.n_layers))
+    for layer in layers:
+        model.add_hook(
+            utils.get_act_name('resid_pre', layer),
+            partial(sae_hook, subject=subject, strength=strength)
+        )
+
+    yield
+
+    model.reset_hooks()
+
+
+batched_chat = [
+    [
+        {"role": "user",
+         "content": "What book is Hermione from?"}
+    ]
+]
+
+tokens = model.tokenizer.apply_chat_template(
+    batched_chat,
+    padding=True,
+    tokenize=True,
+    return_tensors="pt"
+)
+print(tokens)
+
+for i in range(2):
+    if i == 0:
+        print("steering")
+        with steering(subject="harry potter10", strength=-5):
+            with torch.set_grad_enabled(False):
+                batch_output = model.generate(tokens, max_new_tokens=256)
+                response_tokens = []
+                for prompt, combined in zip(tokens, batch_output):
+                    response = combined[len(prompt):]
+                    response_tokens.append(response)
+
+                responses = model.tokenizer.batch_decode(response_tokens, skip_special_tokens=True)
+
+    else:
+        print("no steering")
+        with torch.set_grad_enabled(False):
+            batch_output = model.generate(tokens, max_new_tokens=256)
+            response_tokens = []
+            for prompt, combined in zip(tokens, batch_output):
+                response = combined[len(prompt):]
+                response_tokens.append(response)
+
+            responses = model.tokenizer.batch_decode(response_tokens, skip_special_tokens=True)
+
+    print(responses[0])