Compare commits

..

10 Commits

Author SHA1 Message Date
ydshieh 5bebf1e9bb Update README.md (#3)
- Update README.md (ba08384a5ec82e3ee3bc26f0740af75496d33843)


Co-authored-by: Mishig Davaadorj <mishig@users.noreply.huggingface.co>
2022-09-16 15:06:54 +00:00
ydshieh d273135c52 Update README.md (#2)
- Update README.md (a6aa67c8207fae9a69d512ded9ffc43a2cebd529)


Co-authored-by: Nicolas Patry <Narsil@users.noreply.huggingface.co>
2022-09-16 11:50:02 +00:00
ydshieh 100350d7e6 Add `feature_extractor_type` (#1)
- Add `feature_extractor_type` (468ade778c7dfc810004d424c08f47a3b93c2001)


Co-authored-by: Olivier Dehaene <olivierdehaene@users.noreply.huggingface.co>
2022-08-31 17:41:54 +00:00
Yih-Dar SHIEH 65636df6de Upload tf_model.h5 with git-lfs 2022-01-09 15:53:19 +00:00
Yih-Dar SHIEH 395ab07079 Upload tf_model.h5 with git-lfs 2021-11-14 10:58:43 +00:00
Yih-Dar SHIEH b0acf67ee3 Update README.md 2021-11-10 18:59:24 +00:00
Yih-Dar SHIEH ea737d2858 Update README.md 2021-10-25 10:04:59 +00:00
Yih-Dar SHIEH 973da61a93 Update README.md 2021-10-25 09:07:40 +00:00
Yih-Dar SHIEH f6d2d95250 Update README.md 2021-10-25 09:05:21 +00:00
Yih-Dar SHIEH 462822078f Update pipeline.py 2021-10-25 08:30:24 +00:00
4 changed files with 84 additions and 18 deletions

View File

@ -1,7 +1,11 @@
---
tags:
- image-classification
library_name: generic
- image-to-text
widget:
- src: https://huggingface.co/datasets/mishig/sample_images/resolve/main/football-match.jpg
example_title: Football Match
- src: https://huggingface.co/datasets/mishig/sample_images/resolve/main/dog-cat.jpg
example_title: Dog & Cat
---
## Example
@ -12,34 +16,88 @@ as a proof-of-concept for the 🤗 FlaxVisionEncoderDecoder Framework.
The model can be used as follows:
**In PyTorch**
```python
import torch
import requests
from PIL import Image
from transformers import ViTFeatureExtractor, AutoTokenizer, VisionEncoderDecoderModel
loc = "ydshieh/vit-gpt2-coco-en"
feature_extractor = ViTFeatureExtractor.from_pretrained(loc)
tokenizer = AutoTokenizer.from_pretrained(loc)
model = VisionEncoderDecoderModel.from_pretrained(loc)
model.eval()
def predict(image):
pixel_values = feature_extractor(images=image, return_tensors="pt").pixel_values
with torch.no_grad():
output_ids = model.generate(pixel_values, max_length=16, num_beams=4, return_dict_in_generate=True).sequences
preds = tokenizer.batch_decode(output_ids, skip_special_tokens=True)
preds = [pred.strip() for pred in preds]
return preds
# We will verify our results on an image of cute cats
url = "http://images.cocodataset.org/val2017/000000039769.jpg"
with Image.open(requests.get(url, stream=True).raw) as image:
preds = predict(image)
print(preds)
# should produce
# ['a cat laying on top of a couch next to another cat']
```
**In Flax**
```python
import jax
import requests
from PIL import Image
from transformers import ViTFeatureExtractor, AutoTokenizer, FlaxVisionEncoderDecoderModel
loc = "ydshieh/vit-gpt2-coco-en"
feature_extractor = ViTFeatureExtractor.from_pretrained(loc)
tokenizer = AutoTokenizer.from_pretrained(loc)
model = FlaxVisionEncoderDecoderModel.from_pretrained(loc)
# We will verify our results on an image of cute cats
url = "http://images.cocodataset.org/val2017/000000039769.jpg"
with Image.open(requests.get(url, stream=True).raw) as img:
pixel_values = feature_extractor(images=img, return_tensors="np").pixel_values
gen_kwargs = {"max_length": 16, "num_beams": 4}
def generate_step(pixel_values):
output_ids = model.generate(pixel_values, max_length=16, num_beams=4).sequences
# This takes sometime when compiling the first time, but the subsequent inference will be much faster
@jax.jit
def generate(pixel_values):
output_ids = model.generate(pixel_values, **gen_kwargs).sequences
return output_ids
def predict(image):
pixel_values = feature_extractor(images=image, return_tensors="np").pixel_values
output_ids = generate(pixel_values)
preds = tokenizer.batch_decode(output_ids, skip_special_tokens=True)
preds = [pred.strip() for pred in preds]
return preds
preds = generate_step(pixel_values)
print(preds)
# We will verify our results on an image of cute cats
url = "http://images.cocodataset.org/val2017/000000039769.jpg"
with Image.open(requests.get(url, stream=True).raw) as image:
preds = predict(image)
print(preds)
# should produce
# ['a cat laying on top of a couch next to another cat']

View File

@ -20,7 +20,7 @@ class PreTrainedPipeline():
max_length = 16
num_beams = 4
# self.gen_kwargs = {"max_length": max_length, "num_beams": num_beams}
self.gen_kwargs = {"max_length": max_length, "num_beams": num_beams, "return_dict_in_generate": True}
self.gen_kwargs = {"max_length": max_length, "num_beams": num_beams, "return_dict_in_generate": True, "output_scores": True}
self.model.to("cpu")
self.model.eval()
@ -29,8 +29,12 @@ class PreTrainedPipeline():
def _generate(pixel_values):
with torch.no_grad():
output_ids = self.model.generate(pixel_values, **self.gen_kwargs).sequences
return output_ids
outputs = self.model.generate(pixel_values, **self.gen_kwargs)
output_ids = outputs.sequences
sequences_scores = outputs.sequences_scores
return output_ids, sequences_scores
self.generate = _generate
@ -49,10 +53,10 @@ class PreTrainedPipeline():
# pixel_values = self.feature_extractor(images=inputs, return_tensors="np").pixel_values
pixel_values = self.feature_extractor(images=inputs, return_tensors="pt").pixel_values
output_ids = self.generate(pixel_values)
output_ids, sequences_scores = self.generate(pixel_values)
preds = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)
preds = [pred.strip() for pred in preds]
preds = [{"label": preds[0], "score": 1.0}]
preds = [{"label": preds[0], "score": float(sequences_scores[0])}]
return preds

View File

@ -1,6 +1,7 @@
{
"do_normalize": true,
"do_resize": true,
"feature_extractor_type": "ViTFeatureExtractor",
"image_mean": [
0.5,
0.5,

BIN
tf_model.h5 (Stored with Git LFS) Normal file

Binary file not shown.