Spaces:
Running
Running
Commit
·
8334aa7
1
Parent(s):
aac5437
improve prompt
Browse files
app.py
CHANGED
|
@@ -5,24 +5,32 @@ from haystack.nodes import TransformersImageToText
|
|
| 5 |
from haystack.nodes import PromptNode, PromptTemplate
|
| 6 |
from haystack import Pipeline
|
| 7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
image_to_text = TransformersImageToText(
|
| 9 |
model_name_or_path="nlpconnect/vit-gpt2-image-captioning",
|
| 10 |
-
use_gpu=True,
|
| 11 |
-
batch_size=16,
|
| 12 |
progress_bar=True
|
| 13 |
)
|
| 14 |
|
| 15 |
prompt_template = PromptTemplate(prompt="""
|
| 16 |
You will receive a describing text of a photo.
|
| 17 |
-
Try to come up with a nice Instagram caption
|
| 18 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
Describing text:{documents};
|
| 20 |
Caption:
|
| 21 |
""")
|
| 22 |
|
| 23 |
-
# prompt_node = PromptNode(model_name_or_path="gpt-3.5-turbo", api_key=api_key, default_prompt_template=pt)
|
| 24 |
hf_api_key = os.environ["HF_API_KEY"]
|
| 25 |
-
# prompt_node = PromptNode(model_name_or_path="google/flan-t5-large", default_prompt_template=prompt_template)
|
| 26 |
prompt_node = PromptNode(model_name_or_path="tiiuae/falcon-7b-instruct", api_key=hf_api_key, default_prompt_template=prompt_template, model_kwargs={"trust_remote_code":True})
|
| 27 |
|
| 28 |
captioning_pipeline = Pipeline()
|
|
@@ -30,9 +38,6 @@ captioning_pipeline.add_node(component=image_to_text, name="image_to_text", inpu
|
|
| 30 |
captioning_pipeline.add_node(component=prompt_node, name="prompt_node", inputs=["image_to_text"])
|
| 31 |
|
| 32 |
def generate_caption(image_file_paths):
|
| 33 |
-
print(image_file_paths)
|
| 34 |
-
# documents = image_to_text.generate_captions(image_file_paths=[image_file_paths])
|
| 35 |
-
# print(documents[0].content)
|
| 36 |
caption = captioning_pipeline.run(file_paths=[image_file_paths])
|
| 37 |
print(caption)
|
| 38 |
return caption["results"][0]
|
|
|
|
| 5 |
from haystack.nodes import PromptNode, PromptTemplate
|
| 6 |
from haystack import Pipeline
|
| 7 |
|
| 8 |
+
description = """
|
| 9 |
+
# Captionate ✨ 📸
|
| 10 |
+
## Create Instagram captions for your insta pics!
|
| 11 |
+
|
| 12 |
+
Built by [Bilge Yucel](https://twitter.com/bilgeycl) using with [Haystack](https://github.com/deepset-ai/haystack).
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
image_to_text = TransformersImageToText(
|
| 16 |
model_name_or_path="nlpconnect/vit-gpt2-image-captioning",
|
|
|
|
|
|
|
| 17 |
progress_bar=True
|
| 18 |
)
|
| 19 |
|
| 20 |
prompt_template = PromptTemplate(prompt="""
|
| 21 |
You will receive a describing text of a photo.
|
| 22 |
+
Try to come up with a nice Instagram caption.
|
| 23 |
|
| 24 |
+
Requirements for the caption:
|
| 25 |
+
* Must rhyme with the describing text
|
| 26 |
+
* Should be at least 10 words
|
| 27 |
+
* Needs to include one emoji and suitable hastags
|
| 28 |
+
|
| 29 |
Describing text:{documents};
|
| 30 |
Caption:
|
| 31 |
""")
|
| 32 |
|
|
|
|
| 33 |
hf_api_key = os.environ["HF_API_KEY"]
|
|
|
|
| 34 |
prompt_node = PromptNode(model_name_or_path="tiiuae/falcon-7b-instruct", api_key=hf_api_key, default_prompt_template=prompt_template, model_kwargs={"trust_remote_code":True})
|
| 35 |
|
| 36 |
captioning_pipeline = Pipeline()
|
|
|
|
| 38 |
captioning_pipeline.add_node(component=prompt_node, name="prompt_node", inputs=["image_to_text"])
|
| 39 |
|
| 40 |
def generate_caption(image_file_paths):
|
|
|
|
|
|
|
|
|
|
| 41 |
caption = captioning_pipeline.run(file_paths=[image_file_paths])
|
| 42 |
print(caption)
|
| 43 |
return caption["results"][0]
|