A semantic image search engine that indexes photos using CLIP embeddings, then returns the best matching images for any natural language query — no labels needed.
CLIP (Contrastive Language-Image Pretraining) maps images and text into the same vector space. Similar concepts are close together.
from transformers import CLIPProcessor, CLIPModel
from PIL import Image
import torch
# Download CLIP (~600 MB on first run)
model = CLIPModel.from_pretrained('openai/clip-vit-base-patch32')
processor = CLIPProcessor.from_pretrained('openai/clip-vit-base-patch32')
# Compare an image to text descriptions
image = Image.open('dog.jpg').convert('RGB')
texts = ['a dog', 'a cat', 'a car', 'a mountain']
inputs = processor(text=texts, images=image, return_tensors='pt', padding=True)
with torch.no_grad():
outputs = model(**inputs)
# Higher score = better match
probs = outputs.logits_per_image.softmax(dim=1)
for text, prob in zip(texts, probs[0]):
print(f"{text:15} {prob.item():.2%}")a dog 86.3% a cat 8.1% a car 3.2% a mountain 2.4%
Pre-compute embeddings for all images and save them — then search at interactive speed.
import os, json
import numpy as np
from pathlib import Path
IMAGE_DIR = Path('my_photos')
INDEX_FILE = 'image_index.npy'
PATHS_FILE = 'image_paths.json'
def index_images(image_dir):
paths = list(Path(image_dir).rglob('*.jpg')) + \
list(Path(image_dir).rglob('*.png')) + \
list(Path(image_dir).rglob('*.jpeg'))
print(f"Indexing {len(paths)} images...")
embeddings = []
for i, path in enumerate(paths):
try:
img = Image.open(path).convert('RGB')
inputs = processor(images=img, return_tensors='pt', padding=True)
with torch.no_grad():
feat = model.get_image_features(**inputs)
feat = feat / feat.norm(dim=-1, keepdim=True) # normalise
embeddings.append(feat[0].numpy())
if (i+1) % 10 == 0:
print(f" {i+1}/{len(paths)} done")
except Exception as e:
print(f" Skipped {path}: {e}")
np.save(INDEX_FILE, np.array(embeddings))
with open(PATHS_FILE,'w') as f:
json.dump([str(p) for p in paths], f)
print(f"Index saved! {len(embeddings)} images indexed.")
index_images(IMAGE_DIR)Embed the search query with CLIP and find the closest image embeddings using cosine similarity.
import json
import numpy as np
def search_images(query, top_k=5):
# Load index
embeddings = np.load(INDEX_FILE)
with open(PATHS_FILE) as f:
paths = json.load(f)
# Embed the text query
inputs = processor(text=[query], return_tensors='pt', padding=True)
with torch.no_grad():
text_feat = model.get_text_features(**inputs)
text_feat = text_feat / text_feat.norm(dim=-1, keepdim=True)
text_vec = text_feat[0].numpy()
# Cosine similarity
scores = embeddings @ text_vec
top_idx = scores.argsort()[::-1][:top_k]
print(f"\nTop {top_k} results for: '{query}'\n")
for rank, idx in enumerate(top_idx, 1):
print(f" {rank}. {paths[idx]} (score: {scores[idx]:.4f})")
return [paths[i] for i in top_idx]
search_images("a sunset over the ocean")
search_images("people eating food at a table")
search_images("dog playing in the grass")You built semantic image search that works on your own private photo library with no cloud uploads and no labels. This is the same technology behind Google Photos "search by what you remember" and Pinterest visual search.
Spotted a bug, broken code, or something that doesn't look right? Tell us what's off and we'll fix it.