1108 lines
58 KiB
Plaintext
1108 lines
58 KiB
Plaintext
|
{
|
|||
|
|
"nbformat": 4,
|
|||
|
|
"nbformat_minor": 0,
|
|||
|
|
"metadata": {
|
|||
|
|
"colab": {
|
|||
|
|
"name": "Prompt Engineering for ImageNet.ipynb",
|
|||
|
|
"provenance": [],
|
|||
|
|
"collapsed_sections": []
|
|||
|
|
},
|
|||
|
|
"kernelspec": {
|
|||
|
|
"name": "python3",
|
|||
|
|
"display_name": "Python 3"
|
|||
|
|
},
|
|||
|
|
"accelerator": "GPU",
|
|||
|
|
"widgets": {
|
|||
|
|
"application/vnd.jupyter.widget-state+json": {
|
|||
|
|
"66a1639713ae441d8a9b873381f9d774": {
|
|||
|
|
"model_module": "@jupyter-widgets/controls",
|
|||
|
|
"model_name": "HBoxModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "HBoxView",
|
|||
|
|
"_dom_classes": [],
|
|||
|
|
"_model_name": "HBoxModel",
|
|||
|
|
"_view_module": "@jupyter-widgets/controls",
|
|||
|
|
"_model_module_version": "1.5.0",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"_view_module_version": "1.5.0",
|
|||
|
|
"box_style": "",
|
|||
|
|
"layout": "IPY_MODEL_610b775178c645e2b4663b77cc0c67b6",
|
|||
|
|
"_model_module": "@jupyter-widgets/controls",
|
|||
|
|
"children": [
|
|||
|
|
"IPY_MODEL_412dd15f0d8542f5ab2730f8616fb582",
|
|||
|
|
"IPY_MODEL_5e6315f36b4e4eeea5c6294b024e0c97"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"610b775178c645e2b4663b77cc0c67b6": {
|
|||
|
|
"model_module": "@jupyter-widgets/base",
|
|||
|
|
"model_name": "LayoutModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "LayoutView",
|
|||
|
|
"grid_template_rows": null,
|
|||
|
|
"right": null,
|
|||
|
|
"justify_content": null,
|
|||
|
|
"_view_module": "@jupyter-widgets/base",
|
|||
|
|
"overflow": null,
|
|||
|
|
"_model_module_version": "1.2.0",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"flex_flow": null,
|
|||
|
|
"width": null,
|
|||
|
|
"min_width": null,
|
|||
|
|
"border": null,
|
|||
|
|
"align_items": null,
|
|||
|
|
"bottom": null,
|
|||
|
|
"_model_module": "@jupyter-widgets/base",
|
|||
|
|
"top": null,
|
|||
|
|
"grid_column": null,
|
|||
|
|
"overflow_y": null,
|
|||
|
|
"overflow_x": null,
|
|||
|
|
"grid_auto_flow": null,
|
|||
|
|
"grid_area": null,
|
|||
|
|
"grid_template_columns": null,
|
|||
|
|
"flex": null,
|
|||
|
|
"_model_name": "LayoutModel",
|
|||
|
|
"justify_items": null,
|
|||
|
|
"grid_row": null,
|
|||
|
|
"max_height": null,
|
|||
|
|
"align_content": null,
|
|||
|
|
"visibility": null,
|
|||
|
|
"align_self": null,
|
|||
|
|
"height": null,
|
|||
|
|
"min_height": null,
|
|||
|
|
"padding": null,
|
|||
|
|
"grid_auto_rows": null,
|
|||
|
|
"grid_gap": null,
|
|||
|
|
"max_width": null,
|
|||
|
|
"order": null,
|
|||
|
|
"_view_module_version": "1.2.0",
|
|||
|
|
"grid_template_areas": null,
|
|||
|
|
"object_position": null,
|
|||
|
|
"object_fit": null,
|
|||
|
|
"grid_auto_columns": null,
|
|||
|
|
"margin": null,
|
|||
|
|
"display": null,
|
|||
|
|
"left": null
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"412dd15f0d8542f5ab2730f8616fb582": {
|
|||
|
|
"model_module": "@jupyter-widgets/controls",
|
|||
|
|
"model_name": "FloatProgressModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "ProgressView",
|
|||
|
|
"style": "IPY_MODEL_085d5388abda4202bfa66d0c088452f8",
|
|||
|
|
"_dom_classes": [],
|
|||
|
|
"description": "100%",
|
|||
|
|
"_model_name": "FloatProgressModel",
|
|||
|
|
"bar_style": "success",
|
|||
|
|
"max": 1000,
|
|||
|
|
"_view_module": "@jupyter-widgets/controls",
|
|||
|
|
"_model_module_version": "1.5.0",
|
|||
|
|
"value": 1000,
|
|||
|
|
"_view_count": null,
|
|||
|
|
"_view_module_version": "1.5.0",
|
|||
|
|
"orientation": "horizontal",
|
|||
|
|
"min": 0,
|
|||
|
|
"description_tooltip": null,
|
|||
|
|
"_model_module": "@jupyter-widgets/controls",
|
|||
|
|
"layout": "IPY_MODEL_f75124b64aa147c693c67a78f8e3a231"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"5e6315f36b4e4eeea5c6294b024e0c97": {
|
|||
|
|
"model_module": "@jupyter-widgets/controls",
|
|||
|
|
"model_name": "HTMLModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "HTMLView",
|
|||
|
|
"style": "IPY_MODEL_6e5676a054874243b55fc6d120a07d01",
|
|||
|
|
"_dom_classes": [],
|
|||
|
|
"description": "",
|
|||
|
|
"_model_name": "HTMLModel",
|
|||
|
|
"placeholder": "",
|
|||
|
|
"_view_module": "@jupyter-widgets/controls",
|
|||
|
|
"_model_module_version": "1.5.0",
|
|||
|
|
"value": " 1000/1000 [16:51<00:00, 1.01s/it]",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"_view_module_version": "1.5.0",
|
|||
|
|
"description_tooltip": null,
|
|||
|
|
"_model_module": "@jupyter-widgets/controls",
|
|||
|
|
"layout": "IPY_MODEL_dc6d1416c01a4047935ee15c3fd2eb1c"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"085d5388abda4202bfa66d0c088452f8": {
|
|||
|
|
"model_module": "@jupyter-widgets/controls",
|
|||
|
|
"model_name": "ProgressStyleModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "StyleView",
|
|||
|
|
"_model_name": "ProgressStyleModel",
|
|||
|
|
"description_width": "initial",
|
|||
|
|
"_view_module": "@jupyter-widgets/base",
|
|||
|
|
"_model_module_version": "1.5.0",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"_view_module_version": "1.2.0",
|
|||
|
|
"bar_color": null,
|
|||
|
|
"_model_module": "@jupyter-widgets/controls"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"f75124b64aa147c693c67a78f8e3a231": {
|
|||
|
|
"model_module": "@jupyter-widgets/base",
|
|||
|
|
"model_name": "LayoutModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "LayoutView",
|
|||
|
|
"grid_template_rows": null,
|
|||
|
|
"right": null,
|
|||
|
|
"justify_content": null,
|
|||
|
|
"_view_module": "@jupyter-widgets/base",
|
|||
|
|
"overflow": null,
|
|||
|
|
"_model_module_version": "1.2.0",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"flex_flow": null,
|
|||
|
|
"width": null,
|
|||
|
|
"min_width": null,
|
|||
|
|
"border": null,
|
|||
|
|
"align_items": null,
|
|||
|
|
"bottom": null,
|
|||
|
|
"_model_module": "@jupyter-widgets/base",
|
|||
|
|
"top": null,
|
|||
|
|
"grid_column": null,
|
|||
|
|
"overflow_y": null,
|
|||
|
|
"overflow_x": null,
|
|||
|
|
"grid_auto_flow": null,
|
|||
|
|
"grid_area": null,
|
|||
|
|
"grid_template_columns": null,
|
|||
|
|
"flex": null,
|
|||
|
|
"_model_name": "LayoutModel",
|
|||
|
|
"justify_items": null,
|
|||
|
|
"grid_row": null,
|
|||
|
|
"max_height": null,
|
|||
|
|
"align_content": null,
|
|||
|
|
"visibility": null,
|
|||
|
|
"align_self": null,
|
|||
|
|
"height": null,
|
|||
|
|
"min_height": null,
|
|||
|
|
"padding": null,
|
|||
|
|
"grid_auto_rows": null,
|
|||
|
|
"grid_gap": null,
|
|||
|
|
"max_width": null,
|
|||
|
|
"order": null,
|
|||
|
|
"_view_module_version": "1.2.0",
|
|||
|
|
"grid_template_areas": null,
|
|||
|
|
"object_position": null,
|
|||
|
|
"object_fit": null,
|
|||
|
|
"grid_auto_columns": null,
|
|||
|
|
"margin": null,
|
|||
|
|
"display": null,
|
|||
|
|
"left": null
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"6e5676a054874243b55fc6d120a07d01": {
|
|||
|
|
"model_module": "@jupyter-widgets/controls",
|
|||
|
|
"model_name": "DescriptionStyleModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "StyleView",
|
|||
|
|
"_model_name": "DescriptionStyleModel",
|
|||
|
|
"description_width": "",
|
|||
|
|
"_view_module": "@jupyter-widgets/base",
|
|||
|
|
"_model_module_version": "1.5.0",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"_view_module_version": "1.2.0",
|
|||
|
|
"_model_module": "@jupyter-widgets/controls"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"dc6d1416c01a4047935ee15c3fd2eb1c": {
|
|||
|
|
"model_module": "@jupyter-widgets/base",
|
|||
|
|
"model_name": "LayoutModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "LayoutView",
|
|||
|
|
"grid_template_rows": null,
|
|||
|
|
"right": null,
|
|||
|
|
"justify_content": null,
|
|||
|
|
"_view_module": "@jupyter-widgets/base",
|
|||
|
|
"overflow": null,
|
|||
|
|
"_model_module_version": "1.2.0",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"flex_flow": null,
|
|||
|
|
"width": null,
|
|||
|
|
"min_width": null,
|
|||
|
|
"border": null,
|
|||
|
|
"align_items": null,
|
|||
|
|
"bottom": null,
|
|||
|
|
"_model_module": "@jupyter-widgets/base",
|
|||
|
|
"top": null,
|
|||
|
|
"grid_column": null,
|
|||
|
|
"overflow_y": null,
|
|||
|
|
"overflow_x": null,
|
|||
|
|
"grid_auto_flow": null,
|
|||
|
|
"grid_area": null,
|
|||
|
|
"grid_template_columns": null,
|
|||
|
|
"flex": null,
|
|||
|
|
"_model_name": "LayoutModel",
|
|||
|
|
"justify_items": null,
|
|||
|
|
"grid_row": null,
|
|||
|
|
"max_height": null,
|
|||
|
|
"align_content": null,
|
|||
|
|
"visibility": null,
|
|||
|
|
"align_self": null,
|
|||
|
|
"height": null,
|
|||
|
|
"min_height": null,
|
|||
|
|
"padding": null,
|
|||
|
|
"grid_auto_rows": null,
|
|||
|
|
"grid_gap": null,
|
|||
|
|
"max_width": null,
|
|||
|
|
"order": null,
|
|||
|
|
"_view_module_version": "1.2.0",
|
|||
|
|
"grid_template_areas": null,
|
|||
|
|
"object_position": null,
|
|||
|
|
"object_fit": null,
|
|||
|
|
"grid_auto_columns": null,
|
|||
|
|
"margin": null,
|
|||
|
|
"display": null,
|
|||
|
|
"left": null
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"84f80a7f3e764346969a347b0f71b24e": {
|
|||
|
|
"model_module": "@jupyter-widgets/controls",
|
|||
|
|
"model_name": "HBoxModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "HBoxView",
|
|||
|
|
"_dom_classes": [],
|
|||
|
|
"_model_name": "HBoxModel",
|
|||
|
|
"_view_module": "@jupyter-widgets/controls",
|
|||
|
|
"_model_module_version": "1.5.0",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"_view_module_version": "1.5.0",
|
|||
|
|
"box_style": "",
|
|||
|
|
"layout": "IPY_MODEL_392656f01b2945f3bd7903783ed8cc96",
|
|||
|
|
"_model_module": "@jupyter-widgets/controls",
|
|||
|
|
"children": [
|
|||
|
|
"IPY_MODEL_8e47a435519b4ce090879b4be2f61f99",
|
|||
|
|
"IPY_MODEL_41b1ed6b0a9745c1a595377670b15ff4"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"392656f01b2945f3bd7903783ed8cc96": {
|
|||
|
|
"model_module": "@jupyter-widgets/base",
|
|||
|
|
"model_name": "LayoutModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "LayoutView",
|
|||
|
|
"grid_template_rows": null,
|
|||
|
|
"right": null,
|
|||
|
|
"justify_content": null,
|
|||
|
|
"_view_module": "@jupyter-widgets/base",
|
|||
|
|
"overflow": null,
|
|||
|
|
"_model_module_version": "1.2.0",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"flex_flow": null,
|
|||
|
|
"width": null,
|
|||
|
|
"min_width": null,
|
|||
|
|
"border": null,
|
|||
|
|
"align_items": null,
|
|||
|
|
"bottom": null,
|
|||
|
|
"_model_module": "@jupyter-widgets/base",
|
|||
|
|
"top": null,
|
|||
|
|
"grid_column": null,
|
|||
|
|
"overflow_y": null,
|
|||
|
|
"overflow_x": null,
|
|||
|
|
"grid_auto_flow": null,
|
|||
|
|
"grid_area": null,
|
|||
|
|
"grid_template_columns": null,
|
|||
|
|
"flex": null,
|
|||
|
|
"_model_name": "LayoutModel",
|
|||
|
|
"justify_items": null,
|
|||
|
|
"grid_row": null,
|
|||
|
|
"max_height": null,
|
|||
|
|
"align_content": null,
|
|||
|
|
"visibility": null,
|
|||
|
|
"align_self": null,
|
|||
|
|
"height": null,
|
|||
|
|
"min_height": null,
|
|||
|
|
"padding": null,
|
|||
|
|
"grid_auto_rows": null,
|
|||
|
|
"grid_gap": null,
|
|||
|
|
"max_width": null,
|
|||
|
|
"order": null,
|
|||
|
|
"_view_module_version": "1.2.0",
|
|||
|
|
"grid_template_areas": null,
|
|||
|
|
"object_position": null,
|
|||
|
|
"object_fit": null,
|
|||
|
|
"grid_auto_columns": null,
|
|||
|
|
"margin": null,
|
|||
|
|
"display": null,
|
|||
|
|
"left": null
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"8e47a435519b4ce090879b4be2f61f99": {
|
|||
|
|
"model_module": "@jupyter-widgets/controls",
|
|||
|
|
"model_name": "FloatProgressModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "ProgressView",
|
|||
|
|
"style": "IPY_MODEL_179b8ae1eb7f4a828f953e889b141725",
|
|||
|
|
"_dom_classes": [],
|
|||
|
|
"description": "100%",
|
|||
|
|
"_model_name": "FloatProgressModel",
|
|||
|
|
"bar_style": "success",
|
|||
|
|
"max": 313,
|
|||
|
|
"_view_module": "@jupyter-widgets/controls",
|
|||
|
|
"_model_module_version": "1.5.0",
|
|||
|
|
"value": 313,
|
|||
|
|
"_view_count": null,
|
|||
|
|
"_view_module_version": "1.5.0",
|
|||
|
|
"orientation": "horizontal",
|
|||
|
|
"min": 0,
|
|||
|
|
"description_tooltip": null,
|
|||
|
|
"_model_module": "@jupyter-widgets/controls",
|
|||
|
|
"layout": "IPY_MODEL_d8708e8414fd44f4abd6590c9b57996f"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"41b1ed6b0a9745c1a595377670b15ff4": {
|
|||
|
|
"model_module": "@jupyter-widgets/controls",
|
|||
|
|
"model_name": "HTMLModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "HTMLView",
|
|||
|
|
"style": "IPY_MODEL_800e30f5b4f24475a2b0046da0703631",
|
|||
|
|
"_dom_classes": [],
|
|||
|
|
"description": "",
|
|||
|
|
"_model_name": "HTMLModel",
|
|||
|
|
"placeholder": "",
|
|||
|
|
"_view_module": "@jupyter-widgets/controls",
|
|||
|
|
"_model_module_version": "1.5.0",
|
|||
|
|
"value": " 313/313 [02:31<00:00, 2.07it/s]",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"_view_module_version": "1.5.0",
|
|||
|
|
"description_tooltip": null,
|
|||
|
|
"_model_module": "@jupyter-widgets/controls",
|
|||
|
|
"layout": "IPY_MODEL_8764308b948745f1a677332fd21fcaf0"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"179b8ae1eb7f4a828f953e889b141725": {
|
|||
|
|
"model_module": "@jupyter-widgets/controls",
|
|||
|
|
"model_name": "ProgressStyleModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "StyleView",
|
|||
|
|
"_model_name": "ProgressStyleModel",
|
|||
|
|
"description_width": "initial",
|
|||
|
|
"_view_module": "@jupyter-widgets/base",
|
|||
|
|
"_model_module_version": "1.5.0",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"_view_module_version": "1.2.0",
|
|||
|
|
"bar_color": null,
|
|||
|
|
"_model_module": "@jupyter-widgets/controls"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"d8708e8414fd44f4abd6590c9b57996f": {
|
|||
|
|
"model_module": "@jupyter-widgets/base",
|
|||
|
|
"model_name": "LayoutModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "LayoutView",
|
|||
|
|
"grid_template_rows": null,
|
|||
|
|
"right": null,
|
|||
|
|
"justify_content": null,
|
|||
|
|
"_view_module": "@jupyter-widgets/base",
|
|||
|
|
"overflow": null,
|
|||
|
|
"_model_module_version": "1.2.0",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"flex_flow": null,
|
|||
|
|
"width": null,
|
|||
|
|
"min_width": null,
|
|||
|
|
"border": null,
|
|||
|
|
"align_items": null,
|
|||
|
|
"bottom": null,
|
|||
|
|
"_model_module": "@jupyter-widgets/base",
|
|||
|
|
"top": null,
|
|||
|
|
"grid_column": null,
|
|||
|
|
"overflow_y": null,
|
|||
|
|
"overflow_x": null,
|
|||
|
|
"grid_auto_flow": null,
|
|||
|
|
"grid_area": null,
|
|||
|
|
"grid_template_columns": null,
|
|||
|
|
"flex": null,
|
|||
|
|
"_model_name": "LayoutModel",
|
|||
|
|
"justify_items": null,
|
|||
|
|
"grid_row": null,
|
|||
|
|
"max_height": null,
|
|||
|
|
"align_content": null,
|
|||
|
|
"visibility": null,
|
|||
|
|
"align_self": null,
|
|||
|
|
"height": null,
|
|||
|
|
"min_height": null,
|
|||
|
|
"padding": null,
|
|||
|
|
"grid_auto_rows": null,
|
|||
|
|
"grid_gap": null,
|
|||
|
|
"max_width": null,
|
|||
|
|
"order": null,
|
|||
|
|
"_view_module_version": "1.2.0",
|
|||
|
|
"grid_template_areas": null,
|
|||
|
|
"object_position": null,
|
|||
|
|
"object_fit": null,
|
|||
|
|
"grid_auto_columns": null,
|
|||
|
|
"margin": null,
|
|||
|
|
"display": null,
|
|||
|
|
"left": null
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"800e30f5b4f24475a2b0046da0703631": {
|
|||
|
|
"model_module": "@jupyter-widgets/controls",
|
|||
|
|
"model_name": "DescriptionStyleModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "StyleView",
|
|||
|
|
"_model_name": "DescriptionStyleModel",
|
|||
|
|
"description_width": "",
|
|||
|
|
"_view_module": "@jupyter-widgets/base",
|
|||
|
|
"_model_module_version": "1.5.0",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"_view_module_version": "1.2.0",
|
|||
|
|
"_model_module": "@jupyter-widgets/controls"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"8764308b948745f1a677332fd21fcaf0": {
|
|||
|
|
"model_module": "@jupyter-widgets/base",
|
|||
|
|
"model_name": "LayoutModel",
|
|||
|
|
"state": {
|
|||
|
|
"_view_name": "LayoutView",
|
|||
|
|
"grid_template_rows": null,
|
|||
|
|
"right": null,
|
|||
|
|
"justify_content": null,
|
|||
|
|
"_view_module": "@jupyter-widgets/base",
|
|||
|
|
"overflow": null,
|
|||
|
|
"_model_module_version": "1.2.0",
|
|||
|
|
"_view_count": null,
|
|||
|
|
"flex_flow": null,
|
|||
|
|
"width": null,
|
|||
|
|
"min_width": null,
|
|||
|
|
"border": null,
|
|||
|
|
"align_items": null,
|
|||
|
|
"bottom": null,
|
|||
|
|
"_model_module": "@jupyter-widgets/base",
|
|||
|
|
"top": null,
|
|||
|
|
"grid_column": null,
|
|||
|
|
"overflow_y": null,
|
|||
|
|
"overflow_x": null,
|
|||
|
|
"grid_auto_flow": null,
|
|||
|
|
"grid_area": null,
|
|||
|
|
"grid_template_columns": null,
|
|||
|
|
"flex": null,
|
|||
|
|
"_model_name": "LayoutModel",
|
|||
|
|
"justify_items": null,
|
|||
|
|
"grid_row": null,
|
|||
|
|
"max_height": null,
|
|||
|
|
"align_content": null,
|
|||
|
|
"visibility": null,
|
|||
|
|
"align_self": null,
|
|||
|
|
"height": null,
|
|||
|
|
"min_height": null,
|
|||
|
|
"padding": null,
|
|||
|
|
"grid_auto_rows": null,
|
|||
|
|
"grid_gap": null,
|
|||
|
|
"max_width": null,
|
|||
|
|
"order": null,
|
|||
|
|
"_view_module_version": "1.2.0",
|
|||
|
|
"grid_template_areas": null,
|
|||
|
|
"object_position": null,
|
|||
|
|
"object_fit": null,
|
|||
|
|
"grid_auto_columns": null,
|
|||
|
|
"margin": null,
|
|||
|
|
"display": null,
|
|||
|
|
"left": null
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"cells": [
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "53N4k0pj_9qL"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"# Preparation for Colab\n",
|
|||
|
|
"\n",
|
|||
|
|
"Make sure you're running a GPU runtime; if not, select \"GPU\" as the hardware accelerator in Runtime > Change Runtime Type in the menu. The next cells will install the `clip` package and its dependencies, and check if PyTorch 1.7.1 or later is installed."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"metadata": {
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/"
|
|||
|
|
},
|
|||
|
|
"id": "0BpdJkdBssk9",
|
|||
|
|
"outputId": "41a4070f-5321-4fc4-bd4d-0b5c1f476d56"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"! pip install ftfy regex tqdm\n",
|
|||
|
|
"! pip install git+https://github.com/openai/CLIP.git"
|
|||
|
|
],
|
|||
|
|
"execution_count": 1,
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"Collecting ftfy\n",
|
|||
|
|
" Downloading ftfy-6.0.3.tar.gz (64 kB)\n",
|
|||
|
|
"\u001b[?25l\r\u001b[K |█████ | 10 kB 14.9 MB/s eta 0:00:01\r\u001b[K |██████████▏ | 20 kB 18.7 MB/s eta 0:00:01\r\u001b[K |███████████████▎ | 30 kB 9.0 MB/s eta 0:00:01\r\u001b[K |████████████████████▍ | 40 kB 4.1 MB/s eta 0:00:01\r\u001b[K |█████████████████████████▌ | 51 kB 4.6 MB/s eta 0:00:01\r\u001b[K |██████████████████████████████▋ | 61 kB 4.7 MB/s eta 0:00:01\r\u001b[K |████████████████████████████████| 64 kB 1.3 MB/s \n",
|
|||
|
|
"\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (2019.12.20)\n",
|
|||
|
|
"Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (4.41.1)\n",
|
|||
|
|
"Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from ftfy) (0.2.5)\n",
|
|||
|
|
"Building wheels for collected packages: ftfy\n",
|
|||
|
|
" Building wheel for ftfy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
|
|||
|
|
" Created wheel for ftfy: filename=ftfy-6.0.3-py3-none-any.whl size=41934 sha256=90ec193331444b2c4ff1cd81935e7de42065b89d304db7efac67bcfd87c27873\n",
|
|||
|
|
" Stored in directory: /root/.cache/pip/wheels/19/f5/38/273eb3b5e76dfd850619312f693716ac4518b498f5ffb6f56d\n",
|
|||
|
|
"Successfully built ftfy\n",
|
|||
|
|
"Installing collected packages: ftfy\n",
|
|||
|
|
"Successfully installed ftfy-6.0.3\n",
|
|||
|
|
"Collecting git+https://github.com/openai/CLIP.git\n",
|
|||
|
|
" Cloning https://github.com/openai/CLIP.git to /tmp/pip-req-build-hqnbveqi\n",
|
|||
|
|
" Running command git clone -q https://github.com/openai/CLIP.git /tmp/pip-req-build-hqnbveqi\n",
|
|||
|
|
"Requirement already satisfied: ftfy in /usr/local/lib/python3.7/dist-packages (from clip==1.0) (6.0.3)\n",
|
|||
|
|
"Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from clip==1.0) (2019.12.20)\n",
|
|||
|
|
"Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from clip==1.0) (4.41.1)\n",
|
|||
|
|
"Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from clip==1.0) (1.9.0+cu102)\n",
|
|||
|
|
"Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (from clip==1.0) (0.10.0+cu102)\n",
|
|||
|
|
"Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from ftfy->clip==1.0) (0.2.5)\n",
|
|||
|
|
"Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch->clip==1.0) (3.7.4.3)\n",
|
|||
|
|
"Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision->clip==1.0) (1.19.5)\n",
|
|||
|
|
"Requirement already satisfied: pillow>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision->clip==1.0) (7.1.2)\n",
|
|||
|
|
"Building wheels for collected packages: clip\n",
|
|||
|
|
" Building wheel for clip (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
|
|||
|
|
" Created wheel for clip: filename=clip-1.0-py3-none-any.whl size=1369080 sha256=fda43d2b80cfb2b33c2d43e23ea5f53293a9a8b48d5f9e341de527f6adfbf5a3\n",
|
|||
|
|
" Stored in directory: /tmp/pip-ephem-wheel-cache-kmmplf44/wheels/fd/b9/c3/5b4470e35ed76e174bff77c92f91da82098d5e35fd5bc8cdac\n",
|
|||
|
|
"Successfully built clip\n",
|
|||
|
|
"Installing collected packages: clip\n",
|
|||
|
|
"Successfully installed clip-1.0\n"
|
|||
|
|
],
|
|||
|
|
"name": "stdout"
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "C1hkDT38hSaP",
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/"
|
|||
|
|
},
|
|||
|
|
"outputId": "e10d4f17-8fa6-4b75-a18f-f0c38990b5a3"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"import numpy as np\n",
|
|||
|
|
"import torch\n",
|
|||
|
|
"import clip\n",
|
|||
|
|
"from tqdm.notebook import tqdm\n",
|
|||
|
|
"from pkg_resources import packaging\n",
|
|||
|
|
"\n",
|
|||
|
|
"print(\"Torch version:\", torch.__version__)\n"
|
|||
|
|
],
|
|||
|
|
"execution_count": 2,
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"Torch version: 1.9.0+cu102\n"
|
|||
|
|
],
|
|||
|
|
"name": "stdout"
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "eFxgLV5HAEEw"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"# Loading the model\n",
|
|||
|
|
"\n",
|
|||
|
|
"Download and instantiate a CLIP model using the `clip` module that we just installed."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "uLFS29hnhlY4",
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/"
|
|||
|
|
},
|
|||
|
|
"outputId": "09abb234-693e-4efb-953f-e1847ba95758"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"clip.available_models()"
|
|||
|
|
],
|
|||
|
|
"execution_count": 3,
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "execute_result",
|
|||
|
|
"data": {
|
|||
|
|
"text/plain": [
|
|||
|
|
"['RN50', 'RN101', 'RN50x4', 'RN50x16', 'ViT-B/32', 'ViT-B/16']"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {
|
|||
|
|
"tags": []
|
|||
|
|
},
|
|||
|
|
"execution_count": 3
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "cboKZocQlSYX",
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/"
|
|||
|
|
},
|
|||
|
|
"outputId": "240acdd0-ca62-45db-8418-9e4ef73e8aff"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"model, preprocess = clip.load(\"ViT-B/32\")"
|
|||
|
|
],
|
|||
|
|
"execution_count": 4,
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"100%|███████████████████████████████████████| 338M/338M [00:05<00:00, 63.6MiB/s]\n"
|
|||
|
|
],
|
|||
|
|
"name": "stderr"
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"metadata": {
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/"
|
|||
|
|
},
|
|||
|
|
"id": "IBRVTY9lbGm8",
|
|||
|
|
"outputId": "785019a1-1f40-45b0-e349-b0d4ec3173bf"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"input_resolution = model.visual.input_resolution\n",
|
|||
|
|
"context_length = model.context_length\n",
|
|||
|
|
"vocab_size = model.vocab_size\n",
|
|||
|
|
"\n",
|
|||
|
|
"print(\"Model parameters:\", f\"{np.sum([int(np.prod(p.shape)) for p in model.parameters()]):,}\")\n",
|
|||
|
|
"print(\"Input resolution:\", input_resolution)\n",
|
|||
|
|
"print(\"Context length:\", context_length)\n",
|
|||
|
|
"print(\"Vocab size:\", vocab_size)"
|
|||
|
|
],
|
|||
|
|
"execution_count": 5,
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"Model parameters: 151,277,313\n",
|
|||
|
|
"Input resolution: 224\n",
|
|||
|
|
"Context length: 77\n",
|
|||
|
|
"Vocab size: 49408\n"
|
|||
|
|
],
|
|||
|
|
"name": "stdout"
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "LhO3OtOmF8M4"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"# Preparing ImageNet labels and prompts\n",
|
|||
|
|
"\n",
|
|||
|
|
"The following cell contains the 1,000 labels for the ImageNet dataset, followed by the text templates we'll use as \"prompt engineering\"."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "R2HbOZrqa0jF"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"imagenet_classes = [\"tench\", \"goldfish\", \"great white shark\", \"tiger shark\", \"hammerhead shark\", \"electric ray\", \"stingray\", \"rooster\", \"hen\", \"ostrich\", \"brambling\", \"goldfinch\", \"house finch\", \"junco\", \"indigo bunting\", \"American robin\", \"bulbul\", \"jay\", \"magpie\", \"chickadee\", \"American dipper\", \"kite (bird of prey)\", \"bald eagle\", \"vulture\", \"great grey owl\", \"fire salamander\", \"smooth newt\", \"newt\", \"spotted salamander\", \"axolotl\", \"American bullfrog\", \"tree frog\", \"tailed frog\", \"loggerhead sea turtle\", \"leatherback sea turtle\", \"mud turtle\", \"terrapin\", \"box turtle\", \"banded gecko\", \"green iguana\", \"Carolina anole\", \"desert grassland whiptail lizard\", \"agama\", \"frilled-necked lizard\", \"alligator lizard\", \"Gila monster\", \"European green lizard\", \"chameleon\", \"Komodo dragon\", \"Nile crocodile\", \"American alligator\", \"triceratops\", \"worm snake\", \"ring-necked snake\", \"eastern hog-nosed snake\", \"smooth green snake\", \"kingsnake\", \"garter snake\", \"water snake\", \"vine snake\", \"night snake\", \"boa constrictor\", \"African rock python\", \"Indian cobra\", \"green mamba\", \"sea snake\", \"Saharan horned viper\", \"eastern diamondback rattlesnake\", \"sidewinder rattlesnake\", \"trilobite\", \"harvestman\", \"scorpion\", \"yellow garden spider\", \"barn spider\", \"European garden spider\", \"southern black widow\", \"tarantula\", \"wolf spider\", \"tick\", \"centipede\", \"black grouse\", \"ptarmigan\", \"ruffed grouse\", \"prairie grouse\", \"peafowl\", \"quail\", \"partridge\", \"african grey parrot\", \"macaw\", \"sulphur-crested cockatoo\", \"lorikeet\", \"coucal\", \"bee eater\", \"hornbill\", \"hummingbird\", \"jacamar\", \"toucan\", \"duck\", \"red-breasted merganser\", \"goose\", \"black swan\", \"tusker\", \"echidna\", \"platypus\", \"wallaby\", \"koala\", \"wombat\", \"jellyfish\", \"sea anemone\", \"brain coral\", \"flatworm\", \"nematode\", \"conch\", \"snail\", \"slug\", \"sea slug\", \"chiton\", \"chambered nautilus\", \"Dungeness crab\", \"rock crab\", \"fiddler crab\", \"red king crab\", \"American lobster\", \"spiny lobster\", \"crayfish\", \"hermit crab\", \"isopod\", \"white stork\", \"black stork\", \"spoonbill\", \"flamingo\", \"little blue heron\", \"great egret\", \"bittern bird\", \"crane bird\", \"limpkin\", \"common gallinule\", \"American coot\", \"bustard\", \"ruddy turnstone\", \"dunlin\", \"common redshank\", \"dowitcher\", \"oystercatcher\", \"pelican\", \"king penguin\", \"albatross\", \"grey whale\", \"killer whale\", \"dugong\", \"sea lion\", \"Chihuahua\", \"Japanese Chin\", \"Maltese\", \"Pekingese\", \"Shih Tzu\", \"King Charles Spaniel\", \"Papillon\", \"toy terrier\", \"Rhodesian Ridgeback\", \"Afghan Hound\", \"Basset Hound\", \"Beagle\", \"Bloodhound\", \"Bluetick Coonhound\", \"Black and Tan Coonhound\", \"Treeing Walker Coonhound\", \"English foxhound\", \"Redbone Coonhound\", \"borzoi\", \"Irish Wolfhound\", \"Italian Greyhound\", \"Whippet\", \"Ibizan Hound\", \"Norwegian Elkhound\", \"Otterhound\", \"Saluki\", \"Scottish Deerhound\", \"Weimaraner\", \"Staffordshire Bull Terrier\", \"American Staffordshire Terrier\", \"Bedlington Terrier\", \"Border Terrier\", \"Kerry Blue Terrier\", \"Irish Terrier\", \"Norfolk Terrier\", \"Norwich Terrier\", \"Yorkshire Terrier\", \"Wire Fox Terrier\", \"Lakeland Terrier\", \"Sealyham Terrier\", \"Airedale Terrier\", \"Cairn Terrier\", \"Australian Terrier\", \"Dandie Dinmont Terrier\", \"Boston Terrier\", \"Miniature Schnauzer\", \"Giant Schnauzer\", \"Standard Schnauzer\", \"Scottish Terrier\", \"Tibetan Terrier\", \"Australian Silky Terrier\", \"Soft-coated Wheaten Terrier\", \"West Highland White Terrier\", \"Lhasa Apso\", \"Flat-Coated Retriever\", \"Curly-coated Retriever\", \"Golden Retriever\", \"Labrador Retriever\", \"Chesapeake Bay Retriever\", \"German Shorthaired Pointer\", \"Vizsla\", \"English Setter\", \"Irish Setter\", \"Gordon Setter\", \"Brittany dog\", \"Clumber Spaniel\", \"English Springer Spani
|
|||
|
|
],
|
|||
|
|
"execution_count": 6,
|
|||
|
|
"outputs": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "eMQSCuBta2G6"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"A subset of these class names are modified from the default ImageNet class names sourced from Anish Athalye's imagenet-simple-labels.\n",
|
|||
|
|
"\n",
|
|||
|
|
"These edits were made via trial and error and concentrated on the lowest performing classes according to top_1 and top_5 accuracy on the ImageNet training set for the RN50, RN101, and RN50x4 models. These tweaks improve top_1 by 1.5% on ViT-B/32 over using the default class names. Alec got bored somewhere along the way as gains started to diminish and never finished updating / tweaking the list. He also didn't revisit this with the better performing RN50x16, RN50x64, or any of the ViT models. He thinks it's likely another 0.5% to 1% top_1 could be gained from further work here. It'd be interesting to more rigorously study / understand this.\n",
|
|||
|
|
"\n",
|
|||
|
|
"Some examples beyond the crane/crane -> construction crane / bird crane issue mentioned in Section 3.1.4 of the paper include:\n",
|
|||
|
|
"\n",
|
|||
|
|
"- CLIP interprets \"nail\" as \"fingernail\" so we changed the label to \"metal nail\".\n",
|
|||
|
|
"- ImageNet kite class refers to the bird of prey, not the flying toy, so we changed \"kite\" to \"kite (bird of prey)\"\n",
|
|||
|
|
"- The ImageNet class for red wolf seems to include a lot of mislabeled maned wolfs so we changed \"red wolf\" to \"red wolf or maned wolf\""
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "toGtcd-Ji_MD",
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/"
|
|||
|
|
},
|
|||
|
|
"outputId": "b6eb0753-2bee-4144-abe3-fbd23f35f555"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"imagenet_templates = [\n",
|
|||
|
|
" 'a bad photo of a {}.',\n",
|
|||
|
|
" 'a photo of many {}.',\n",
|
|||
|
|
" 'a sculpture of a {}.',\n",
|
|||
|
|
" 'a photo of the hard to see {}.',\n",
|
|||
|
|
" 'a low resolution photo of the {}.',\n",
|
|||
|
|
" 'a rendering of a {}.',\n",
|
|||
|
|
" 'graffiti of a {}.',\n",
|
|||
|
|
" 'a bad photo of the {}.',\n",
|
|||
|
|
" 'a cropped photo of the {}.',\n",
|
|||
|
|
" 'a tattoo of a {}.',\n",
|
|||
|
|
" 'the embroidered {}.',\n",
|
|||
|
|
" 'a photo of a hard to see {}.',\n",
|
|||
|
|
" 'a bright photo of a {}.',\n",
|
|||
|
|
" 'a photo of a clean {}.',\n",
|
|||
|
|
" 'a photo of a dirty {}.',\n",
|
|||
|
|
" 'a dark photo of the {}.',\n",
|
|||
|
|
" 'a drawing of a {}.',\n",
|
|||
|
|
" 'a photo of my {}.',\n",
|
|||
|
|
" 'the plastic {}.',\n",
|
|||
|
|
" 'a photo of the cool {}.',\n",
|
|||
|
|
" 'a close-up photo of a {}.',\n",
|
|||
|
|
" 'a black and white photo of the {}.',\n",
|
|||
|
|
" 'a painting of the {}.',\n",
|
|||
|
|
" 'a painting of a {}.',\n",
|
|||
|
|
" 'a pixelated photo of the {}.',\n",
|
|||
|
|
" 'a sculpture of the {}.',\n",
|
|||
|
|
" 'a bright photo of the {}.',\n",
|
|||
|
|
" 'a cropped photo of a {}.',\n",
|
|||
|
|
" 'a plastic {}.',\n",
|
|||
|
|
" 'a photo of the dirty {}.',\n",
|
|||
|
|
" 'a jpeg corrupted photo of a {}.',\n",
|
|||
|
|
" 'a blurry photo of the {}.',\n",
|
|||
|
|
" 'a photo of the {}.',\n",
|
|||
|
|
" 'a good photo of the {}.',\n",
|
|||
|
|
" 'a rendering of the {}.',\n",
|
|||
|
|
" 'a {} in a video game.',\n",
|
|||
|
|
" 'a photo of one {}.',\n",
|
|||
|
|
" 'a doodle of a {}.',\n",
|
|||
|
|
" 'a close-up photo of the {}.',\n",
|
|||
|
|
" 'a photo of a {}.',\n",
|
|||
|
|
" 'the origami {}.',\n",
|
|||
|
|
" 'the {} in a video game.',\n",
|
|||
|
|
" 'a sketch of a {}.',\n",
|
|||
|
|
" 'a doodle of the {}.',\n",
|
|||
|
|
" 'a origami {}.',\n",
|
|||
|
|
" 'a low resolution photo of a {}.',\n",
|
|||
|
|
" 'the toy {}.',\n",
|
|||
|
|
" 'a rendition of the {}.',\n",
|
|||
|
|
" 'a photo of the clean {}.',\n",
|
|||
|
|
" 'a photo of a large {}.',\n",
|
|||
|
|
" 'a rendition of a {}.',\n",
|
|||
|
|
" 'a photo of a nice {}.',\n",
|
|||
|
|
" 'a photo of a weird {}.',\n",
|
|||
|
|
" 'a blurry photo of a {}.',\n",
|
|||
|
|
" 'a cartoon {}.',\n",
|
|||
|
|
" 'art of a {}.',\n",
|
|||
|
|
" 'a sketch of the {}.',\n",
|
|||
|
|
" 'a embroidered {}.',\n",
|
|||
|
|
" 'a pixelated photo of a {}.',\n",
|
|||
|
|
" 'itap of the {}.',\n",
|
|||
|
|
" 'a jpeg corrupted photo of the {}.',\n",
|
|||
|
|
" 'a good photo of a {}.',\n",
|
|||
|
|
" 'a plushie {}.',\n",
|
|||
|
|
" 'a photo of the nice {}.',\n",
|
|||
|
|
" 'a photo of the small {}.',\n",
|
|||
|
|
" 'a photo of the weird {}.',\n",
|
|||
|
|
" 'the cartoon {}.',\n",
|
|||
|
|
" 'art of the {}.',\n",
|
|||
|
|
" 'a drawing of the {}.',\n",
|
|||
|
|
" 'a photo of the large {}.',\n",
|
|||
|
|
" 'a black and white photo of a {}.',\n",
|
|||
|
|
" 'the plushie {}.',\n",
|
|||
|
|
" 'a dark photo of a {}.',\n",
|
|||
|
|
" 'itap of a {}.',\n",
|
|||
|
|
" 'graffiti of the {}.',\n",
|
|||
|
|
" 'a toy {}.',\n",
|
|||
|
|
" 'itap of my {}.',\n",
|
|||
|
|
" 'a photo of a cool {}.',\n",
|
|||
|
|
" 'a photo of a small {}.',\n",
|
|||
|
|
" 'a tattoo of the {}.',\n",
|
|||
|
|
"]\n",
|
|||
|
|
"\n",
|
|||
|
|
"print(f\"{len(imagenet_classes)} classes, {len(imagenet_templates)} templates\")"
|
|||
|
|
],
|
|||
|
|
"execution_count": 7,
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"1000 classes, 80 templates\n"
|
|||
|
|
],
|
|||
|
|
"name": "stdout"
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "aRB5OzgpHwqQ"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"A similar, intuition-guided trial and error based on the ImageNet training set was used for templates. This list is pretty haphazard and was gradually made / expanded over the course of about a year of the project and was revisited / tweaked every few months. A surprising / weird thing was adding templates intended to help ImageNet-R performance (specifying different possible renditions of an object) improved standard ImageNet accuracy too.\n",
|
|||
|
|
"\n",
|
|||
|
|
"After the 80 templates were \"locked\" for the paper, we ran sequential forward selection over the list of 80 templates. The search terminated after ensembling 7 templates and selected them in the order below.\n",
|
|||
|
|
"\n",
|
|||
|
|
"1. itap of a {}.\n",
|
|||
|
|
"2. a bad photo of the {}.\n",
|
|||
|
|
"3. a origami {}.\n",
|
|||
|
|
"4. a photo of the large {}.\n",
|
|||
|
|
"5. a {} in a video game.\n",
|
|||
|
|
"6. art of the {}.\n",
|
|||
|
|
"7. a photo of the small {}.\n",
|
|||
|
|
"\n",
|
|||
|
|
"Speculating, we think it's interesting to see different scales (large and small), a difficult view (a bad photo), and \"abstract\" versions (origami, video game, art), were all selected for, but we haven't studied this in any detail. This subset performs a bit better than the full 80 ensemble reported in the paper, especially for the smaller models."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "4W8ARJVqBJXs"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"# Loading the Images\n",
|
|||
|
|
"\n",
|
|||
|
|
"The ILSVRC2012 datasets are no longer available for download publicly. We instead download the ImageNet-V2 dataset by [Recht et al.](https://arxiv.org/abs/1902.10811).\n",
|
|||
|
|
"\n",
|
|||
|
|
"If you have the ImageNet dataset downloaded, you can replace the dataset with the official torchvision loader, e.g.:\n",
|
|||
|
|
"\n",
|
|||
|
|
"```python\n",
|
|||
|
|
"images = torchvision.datasets.ImageNet(\"path/to/imagenet\", split='val', transform=preprocess)\n",
|
|||
|
|
"```"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"metadata": {
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/"
|
|||
|
|
},
|
|||
|
|
"id": "moHR4UlHKsDc",
|
|||
|
|
"outputId": "40731297-edc7-4cd0-be75-ed426c8fb005"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"! pip install git+https://github.com/modestyachts/ImageNetV2_pytorch\n",
|
|||
|
|
"\n",
|
|||
|
|
"from imagenetv2_pytorch import ImageNetV2Dataset\n",
|
|||
|
|
"\n",
|
|||
|
|
"images = ImageNetV2Dataset(transform=preprocess)\n",
|
|||
|
|
"loader = torch.utils.data.DataLoader(images, batch_size=32, num_workers=2)"
|
|||
|
|
],
|
|||
|
|
"execution_count": 8,
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"Collecting git+https://github.com/modestyachts/ImageNetV2_pytorch\n",
|
|||
|
|
" Cloning https://github.com/modestyachts/ImageNetV2_pytorch to /tmp/pip-req-build-0kih0kn2\n",
|
|||
|
|
" Running command git clone -q https://github.com/modestyachts/ImageNetV2_pytorch /tmp/pip-req-build-0kih0kn2\n",
|
|||
|
|
"Building wheels for collected packages: imagenetv2-pytorch\n",
|
|||
|
|
" Building wheel for imagenetv2-pytorch (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
|
|||
|
|
" Created wheel for imagenetv2-pytorch: filename=imagenetv2_pytorch-0.1-py3-none-any.whl size=2663 sha256=ac31e0ed9c61afc5e0271eed315d3a82844a79ae64f8ce43fc1c98928cec129f\n",
|
|||
|
|
" Stored in directory: /tmp/pip-ephem-wheel-cache-745b5n1m/wheels/ab/ee/f4/73bce5c7f68d28ce632ef33ae87ce60aaca021eb2b3b31a6fa\n",
|
|||
|
|
"Successfully built imagenetv2-pytorch\n",
|
|||
|
|
"Installing collected packages: imagenetv2-pytorch\n",
|
|||
|
|
"Successfully installed imagenetv2-pytorch-0.1\n",
|
|||
|
|
"Dataset matched-frequency not found on disk, downloading....\n"
|
|||
|
|
],
|
|||
|
|
"name": "stdout"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"100%|██████████| 1.26G/1.26G [01:02<00:00, 20.2MiB/s]\n"
|
|||
|
|
],
|
|||
|
|
"name": "stderr"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"Extracting....\n"
|
|||
|
|
],
|
|||
|
|
"name": "stdout"
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "fz6D-F-Wbrtp"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"# Creating zero-shot classifier weights"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"metadata": {
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/",
|
|||
|
|
"height": 67,
|
|||
|
|
"referenced_widgets": [
|
|||
|
|
"66a1639713ae441d8a9b873381f9d774",
|
|||
|
|
"610b775178c645e2b4663b77cc0c67b6",
|
|||
|
|
"412dd15f0d8542f5ab2730f8616fb582",
|
|||
|
|
"5e6315f36b4e4eeea5c6294b024e0c97",
|
|||
|
|
"085d5388abda4202bfa66d0c088452f8",
|
|||
|
|
"f75124b64aa147c693c67a78f8e3a231",
|
|||
|
|
"6e5676a054874243b55fc6d120a07d01",
|
|||
|
|
"dc6d1416c01a4047935ee15c3fd2eb1c"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"id": "sRqDoz1Gbsii",
|
|||
|
|
"outputId": "312b8ebf-3961-4903-d8cb-3b7a94cc97b6"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"def zeroshot_classifier(classnames, templates):\n",
|
|||
|
|
" with torch.no_grad():\n",
|
|||
|
|
" zeroshot_weights = []\n",
|
|||
|
|
" for classname in tqdm(classnames):\n",
|
|||
|
|
" texts = [template.format(classname) for template in templates] #format with class\n",
|
|||
|
|
" texts = clip.tokenize(texts).cuda() #tokenize\n",
|
|||
|
|
" class_embeddings = model.encode_text(texts) #embed with text encoder\n",
|
|||
|
|
" class_embeddings /= class_embeddings.norm(dim=-1, keepdim=True)\n",
|
|||
|
|
" class_embedding = class_embeddings.mean(dim=0)\n",
|
|||
|
|
" class_embedding /= class_embedding.norm()\n",
|
|||
|
|
" zeroshot_weights.append(class_embedding)\n",
|
|||
|
|
" zeroshot_weights = torch.stack(zeroshot_weights, dim=1).cuda()\n",
|
|||
|
|
" return zeroshot_weights\n",
|
|||
|
|
"\n",
|
|||
|
|
"\n",
|
|||
|
|
"zeroshot_weights = zeroshot_classifier(imagenet_classes, imagenet_templates)"
|
|||
|
|
],
|
|||
|
|
"execution_count": 9,
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "display_data",
|
|||
|
|
"data": {
|
|||
|
|
"application/vnd.jupyter.widget-view+json": {
|
|||
|
|
"model_id": "66a1639713ae441d8a9b873381f9d774",
|
|||
|
|
"version_minor": 0,
|
|||
|
|
"version_major": 2
|
|||
|
|
},
|
|||
|
|
"text/plain": [
|
|||
|
|
"HBox(children=(FloatProgress(value=0.0, max=1000.0), HTML(value='')))"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {
|
|||
|
|
"tags": []
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"\n"
|
|||
|
|
],
|
|||
|
|
"name": "stdout"
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "1fZo7hG8iJP5"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"# Zero-shot prediction"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "j4kPSZoShQxN"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"def accuracy(output, target, topk=(1,)):\n",
|
|||
|
|
" pred = output.topk(max(topk), 1, True, True)[1].t()\n",
|
|||
|
|
" correct = pred.eq(target.view(1, -1).expand_as(pred))\n",
|
|||
|
|
" return [float(correct[:k].reshape(-1).float().sum(0, keepdim=True).cpu().numpy()) for k in topk]"
|
|||
|
|
],
|
|||
|
|
"execution_count": 10,
|
|||
|
|
"outputs": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"metadata": {
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/",
|
|||
|
|
"height": 102,
|
|||
|
|
"referenced_widgets": [
|
|||
|
|
"84f80a7f3e764346969a347b0f71b24e",
|
|||
|
|
"392656f01b2945f3bd7903783ed8cc96",
|
|||
|
|
"8e47a435519b4ce090879b4be2f61f99",
|
|||
|
|
"41b1ed6b0a9745c1a595377670b15ff4",
|
|||
|
|
"179b8ae1eb7f4a828f953e889b141725",
|
|||
|
|
"d8708e8414fd44f4abd6590c9b57996f",
|
|||
|
|
"800e30f5b4f24475a2b0046da0703631",
|
|||
|
|
"8764308b948745f1a677332fd21fcaf0"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"id": "wKJ7YsdlkDXo",
|
|||
|
|
"outputId": "ab824854-38e4-4d37-ad40-2a7ce3c5fd43"
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"with torch.no_grad():\n",
|
|||
|
|
" top1, top5, n = 0., 0., 0.\n",
|
|||
|
|
" for i, (images, target) in enumerate(tqdm(loader)):\n",
|
|||
|
|
" images = images.cuda()\n",
|
|||
|
|
" target = target.cuda()\n",
|
|||
|
|
" \n",
|
|||
|
|
" # predict\n",
|
|||
|
|
" image_features = model.encode_image(images)\n",
|
|||
|
|
" image_features /= image_features.norm(dim=-1, keepdim=True)\n",
|
|||
|
|
" logits = 100. * image_features @ zeroshot_weights\n",
|
|||
|
|
"\n",
|
|||
|
|
" # measure accuracy\n",
|
|||
|
|
" acc1, acc5 = accuracy(logits, target, topk=(1, 5))\n",
|
|||
|
|
" top1 += acc1\n",
|
|||
|
|
" top5 += acc5\n",
|
|||
|
|
" n += images.size(0)\n",
|
|||
|
|
"\n",
|
|||
|
|
"top1 = (top1 / n) * 100\n",
|
|||
|
|
"top5 = (top5 / n) * 100 \n",
|
|||
|
|
"\n",
|
|||
|
|
"print(f\"Top-1 accuracy: {top1:.2f}\")\n",
|
|||
|
|
"print(f\"Top-5 accuracy: {top5:.2f}\")"
|
|||
|
|
],
|
|||
|
|
"execution_count": 11,
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "display_data",
|
|||
|
|
"data": {
|
|||
|
|
"application/vnd.jupyter.widget-view+json": {
|
|||
|
|
"model_id": "84f80a7f3e764346969a347b0f71b24e",
|
|||
|
|
"version_minor": 0,
|
|||
|
|
"version_major": 2
|
|||
|
|
},
|
|||
|
|
"text/plain": [
|
|||
|
|
"HBox(children=(FloatProgress(value=0.0, max=313.0), HTML(value='')))"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {
|
|||
|
|
"tags": []
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"\n",
|
|||
|
|
"Top-1 accuracy: 55.93\n",
|
|||
|
|
"Top-5 accuracy: 83.36\n"
|
|||
|
|
],
|
|||
|
|
"name": "stdout"
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
}
|