photos/src/lib/photo-service.ts
Michael Mainguy 85c1479d94 Add comprehensive AI-powered photo analysis with dual-model classification
## Features Added:
- **Dual Model Classification**: ViT (objects) + CLIP (style/artistic concepts)
- **Image Captioning**: BLIP model for detailed photo descriptions
- **Auto-tagging**: Process all photos with configurable confidence thresholds
- **Tag Management**: Clear all tags functionality with safety confirmations
- **Comprehensive Analysis**: 15-25+ tags per image covering objects, style, mood, lighting

## New API Endpoints:
- `/api/classify/batch` - Batch classification with comprehensive mode
- `/api/classify/comprehensive` - Dual-model analysis for maximum tags
- `/api/classify/config` - Tunable classifier parameters
- `/api/caption/batch` - Batch image captioning
- `/api/tags/clear` - Clear all tags with safety checks

## UI Enhancements:
- Auto-tag All button (processes 5 photos at a time)
- Caption All button (processes 3 photos at a time)
- Clear All Tags button with confirmation dialogs
- Real-time progress bars for batch operations
- Tag pills displayed on thumbnails and image modal
- AI-generated captions shown in image modal

## Performance Optimizations:
- Uses cached thumbnails for 10-100x faster processing
- Parallel model initialization and processing
- Graceful fallback to original files when thumbnails fail
- Configurable batch sizes to prevent memory issues

## Technical Implementation:
- Vision Transformer (ViT) for ImageNet object classification (1000+ classes)
- CLIP for zero-shot artistic/style classification (photography, lighting, mood)
- BLIP for natural language image descriptions
- Comprehensive safety checks and error handling
- Database integration for persistent tag and caption storage

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-27 17:05:54 -05:00

613 lines
20 KiB
TypeScript

import { getDatabase } from './database'
import { Photo, Album, Tag, Directory, ImageHash, PhotoHash } from '@/types/photo'
import { randomUUID } from 'crypto'
export class PhotoService {
private db = getDatabase()
// Photo operations
createPhoto(photoData: Omit<Photo, 'id' | 'indexed_at' | 'thumbnail_blob' | 'thumbnail_size' | 'thumbnail_generated_at'>): Photo {
const id = randomUUID()
const insertPhoto = this.db.prepare(`
INSERT INTO photos (
id, filename, filepath, directory, filesize, created_at, modified_at,
width, height, format, favorite, rating, description, metadata
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`)
try {
// Sanitize each value to ensure it's a primitive type for SQLite
const sanitizeValue = (value: any): string | number | boolean | null => {
// Explicitly handle null and undefined
if (value === null || value === undefined) return null
// Handle primitive types
if (typeof value === 'string') return value
if (typeof value === 'number') return value
if (typeof value === 'boolean') return value
// Handle other types
if (typeof value === 'bigint') return Number(value)
if (value instanceof Date) return value.toISOString()
if (typeof value === 'object') {
// If it's null, return null (redundant but explicit)
if (value === null) return null
// Otherwise stringify the object
return JSON.stringify(value)
}
// Convert everything else to string
return String(value)
}
// Direct approach - build parameters with explicit type checking
const params = [
id, // string
photoData.filename, // string
photoData.filepath, // string
photoData.directory, // string
photoData.filesize, // number
photoData.created_at, // string
photoData.modified_at, // string
photoData.width || null, // number or null
photoData.height || null, // number or null
photoData.format || null, // string or null
photoData.favorite === true ? 1 : 0, // convert boolean to integer
null, // rating - always null for now
null, // description - always null for now
photoData.metadata || null // string or null
]
// Debug: Log each parameter type before database insertion
console.log(`[PHOTO SERVICE] Debug - checking SQL parameters for ${photoData.filename}:`)
const fieldNames = ['id', 'filename', 'filepath', 'directory', 'filesize', 'created_at', 'modified_at', 'width', 'height', 'format', 'favorite', 'rating', 'description', 'metadata']
params.forEach((value, index) => {
const valueStr = value === null ? 'NULL' : (typeof value === 'string' && value.length > 50 ? value.substring(0, 50) + '...' : value)
console.log(` ${fieldNames[index]}:`, typeof value, valueStr)
// Additional check for SQLite compatibility
const isValidSQLiteValue = value === null || typeof value === 'string' || typeof value === 'number' || typeof value === 'bigint'
if (!isValidSQLiteValue) {
console.error(` ❌ INVALID SQLite value for ${fieldNames[index]}:`, value, typeof value)
}
})
// Use spread operator with the clean params array
insertPhoto.run(...params)
return this.getPhoto(id)!
} catch (error) {
console.error(`[PHOTO SERVICE] Error creating photo ${photoData.filename}:`, error)
console.error(`[PHOTO SERVICE] Photo data:`, photoData)
throw error
}
}
getPhoto(id: string): Photo | null {
const selectPhoto = this.db.prepare('SELECT * FROM photos WHERE id = ?')
return selectPhoto.get(id) as Photo | null
}
getPhotoByPath(filepath: string): Photo | null {
const selectPhoto = this.db.prepare('SELECT * FROM photos WHERE filepath = ?')
return selectPhoto.get(filepath) as Photo | null
}
getPhotos(options: {
directory?: string
limit?: number
offset?: number
sortBy?: 'created_at' | 'modified_at' | 'filename' | 'filesize'
sortOrder?: 'ASC' | 'DESC'
favorite?: boolean
rating?: number
} = {}): Photo[] {
let query = 'SELECT * FROM photos WHERE 1=1'
const params: any[] = []
if (options.directory) {
query += ' AND directory = ?'
params.push(options.directory)
}
if (options.favorite !== undefined) {
query += ' AND favorite = ?'
params.push(options.favorite)
}
if (options.rating !== undefined) {
query += ' AND rating = ?'
params.push(options.rating)
}
const sortBy = options.sortBy || 'created_at'
const sortOrder = options.sortOrder || 'DESC'
query += ` ORDER BY ${sortBy} ${sortOrder}`
if (options.limit) {
query += ' LIMIT ?'
params.push(options.limit)
}
if (options.offset) {
query += ' OFFSET ?'
params.push(options.offset)
}
const selectPhotos = this.db.prepare(query)
return selectPhotos.all(...params) as Photo[]
}
updatePhoto(id: string, updates: Partial<Omit<Photo, 'id' | 'filepath' | 'indexed_at'>>): Photo | null {
const fields = Object.keys(updates).filter(key => updates[key as keyof typeof updates] !== undefined)
if (fields.length === 0) return this.getPhoto(id)
const setClause = fields.map(field => `${field} = ?`).join(', ')
const values = fields.map(field => updates[field as keyof typeof updates])
const updatePhoto = this.db.prepare(`UPDATE photos SET ${setClause} WHERE id = ?`)
updatePhoto.run(...values, id)
return this.getPhoto(id)
}
deletePhoto(id: string): boolean {
const deletePhoto = this.db.prepare('DELETE FROM photos WHERE id = ?')
const result = deletePhoto.run(id)
return result.changes > 0
}
// Album operations
createAlbum(albumData: Omit<Album, 'id' | 'created_at' | 'modified_at'>): Album {
const id = randomUUID()
const insertAlbum = this.db.prepare(`
INSERT INTO albums (id, name, description, cover_photo_id)
VALUES (?, ?, ?, ?)
`)
insertAlbum.run(id, albumData.name, albumData.description, albumData.cover_photo_id)
return this.getAlbum(id)!
}
getAlbum(id: string): Album | null {
const selectAlbum = this.db.prepare('SELECT * FROM albums WHERE id = ?')
return selectAlbum.get(id) as Album | null
}
getAlbums(): Album[] {
const selectAlbums = this.db.prepare('SELECT * FROM albums ORDER BY name')
return selectAlbums.all() as Album[]
}
addPhotoToAlbum(photoId: string, albumId: string): boolean {
const insertPhotoAlbum = this.db.prepare(`
INSERT OR IGNORE INTO photo_albums (photo_id, album_id)
VALUES (?, ?)
`)
const result = insertPhotoAlbum.run(photoId, albumId)
return result.changes > 0
}
removePhotoFromAlbum(photoId: string, albumId: string): boolean {
const deletePhotoAlbum = this.db.prepare(`
DELETE FROM photo_albums WHERE photo_id = ? AND album_id = ?
`)
const result = deletePhotoAlbum.run(photoId, albumId)
return result.changes > 0
}
getAlbumPhotos(albumId: string): Photo[] {
const selectAlbumPhotos = this.db.prepare(`
SELECT p.* FROM photos p
JOIN photo_albums pa ON p.id = pa.photo_id
WHERE pa.album_id = ?
ORDER BY pa.added_at DESC
`)
return selectAlbumPhotos.all(albumId) as Photo[]
}
// Tag operations
createTag(tagData: Omit<Tag, 'id' | 'created_at'>): Tag {
const id = randomUUID()
const insertTag = this.db.prepare(`
INSERT INTO tags (id, name, color) VALUES (?, ?, ?)
`)
insertTag.run(id, tagData.name, tagData.color)
return this.getTag(id)!
}
getTag(id: string): Tag | null {
const selectTag = this.db.prepare('SELECT * FROM tags WHERE id = ?')
return selectTag.get(id) as Tag | null
}
getTags(): Tag[] {
const selectTags = this.db.prepare('SELECT * FROM tags ORDER BY name')
return selectTags.all() as Tag[]
}
addPhotoTag(photoId: string, tagId: string): boolean {
const insertPhotoTag = this.db.prepare(`
INSERT OR IGNORE INTO photo_tags (photo_id, tag_id)
VALUES (?, ?)
`)
const result = insertPhotoTag.run(photoId, tagId)
return result.changes > 0
}
removePhotoTag(photoId: string, tagId: string): boolean {
const deletePhotoTag = this.db.prepare(`
DELETE FROM photo_tags WHERE photo_id = ? AND tag_id = ?
`)
const result = deletePhotoTag.run(photoId, tagId)
return result.changes > 0
}
getPhotoTags(photoId: string): Tag[] {
const selectPhotoTags = this.db.prepare(`
SELECT t.* FROM tags t
JOIN photo_tags pt ON t.id = pt.tag_id
WHERE pt.photo_id = ?
ORDER BY t.name
`)
return selectPhotoTags.all(photoId) as Tag[]
}
// Directory operations
createOrUpdateDirectory(directoryData: Omit<Directory, 'id'>): Directory {
const existingDir = this.getDirectoryByPath(directoryData.path)
if (existingDir) {
const updateDir = this.db.prepare(`
UPDATE directories SET name = ?, last_scanned = ?, photo_count = ?, total_size = ?
WHERE path = ?
`)
updateDir.run(
directoryData.name,
directoryData.last_scanned,
directoryData.photo_count,
directoryData.total_size,
directoryData.path
)
return this.getDirectoryByPath(directoryData.path)!
} else {
const id = randomUUID()
const insertDir = this.db.prepare(`
INSERT INTO directories (id, path, name, last_scanned, photo_count, total_size)
VALUES (?, ?, ?, ?, ?, ?)
`)
insertDir.run(
id,
directoryData.path,
directoryData.name,
directoryData.last_scanned,
directoryData.photo_count,
directoryData.total_size
)
return this.getDirectory(id)!
}
}
getDirectory(id: string): Directory | null {
const selectDir = this.db.prepare('SELECT * FROM directories WHERE id = ?')
return selectDir.get(id) as Directory | null
}
getDirectoryByPath(path: string): Directory | null {
const selectDir = this.db.prepare('SELECT * FROM directories WHERE path = ?')
return selectDir.get(path) as Directory | null
}
getDirectories(): Directory[] {
const selectDirs = this.db.prepare('SELECT * FROM directories ORDER BY last_scanned DESC')
return selectDirs.all() as Directory[]
}
deleteDirectory(id: string): boolean {
const deleteDir = this.db.prepare('DELETE FROM directories WHERE id = ?')
const result = deleteDir.run(id)
return result.changes > 0
}
// Statistics
getPhotoCount(): number {
const countPhotos = this.db.prepare('SELECT COUNT(*) as count FROM photos')
return (countPhotos.get() as { count: number }).count
}
getTotalFileSize(): number {
const totalSize = this.db.prepare('SELECT SUM(filesize) as total FROM photos')
return (totalSize.get() as { total: number }).total || 0
}
// Hash operations
createOrUpdateImageHash(sha256Hash: string): ImageHash {
const existingHash = this.getImageHashBySha256(sha256Hash)
if (existingHash) {
// Update file count
const updateHash = this.db.prepare(`
UPDATE image_hashes SET file_count = file_count + 1 WHERE sha256_hash = ?
`)
updateHash.run(sha256Hash)
return this.getImageHashBySha256(sha256Hash)!
} else {
// Create new hash record
const id = randomUUID()
const insertHash = this.db.prepare(`
INSERT INTO image_hashes (id, sha256_hash, file_count) VALUES (?, ?, 1)
`)
insertHash.run(id, sha256Hash)
return this.getImageHash(id)!
}
}
getImageHash(id: string): ImageHash | null {
const selectHash = this.db.prepare('SELECT * FROM image_hashes WHERE id = ?')
return selectHash.get(id) as ImageHash | null
}
getImageHashBySha256(sha256Hash: string): ImageHash | null {
const selectHash = this.db.prepare('SELECT * FROM image_hashes WHERE sha256_hash = ?')
return selectHash.get(sha256Hash) as ImageHash | null
}
associatePhotoWithHash(photoId: string, hashId: string): boolean {
const insertPhotoHash = this.db.prepare(`
INSERT OR IGNORE INTO photo_hashes (photo_id, hash_id) VALUES (?, ?)
`)
const result = insertPhotoHash.run(photoId, hashId)
return result.changes > 0
}
getPhotosByHash(sha256Hash: string): Photo[] {
const selectPhotos = this.db.prepare(`
SELECT p.* FROM photos p
JOIN photo_hashes ph ON p.id = ph.photo_id
JOIN image_hashes ih ON ph.hash_id = ih.id
WHERE ih.sha256_hash = ?
ORDER BY p.created_at DESC
`)
return selectPhotos.all(sha256Hash) as Photo[]
}
getDuplicateHashes(): ImageHash[] {
const selectDuplicates = this.db.prepare(`
SELECT * FROM image_hashes WHERE file_count > 1 ORDER BY file_count DESC
`)
return selectDuplicates.all() as ImageHash[]
}
getPhotosWithDuplicates(): Photo[] {
const selectPhotos = this.db.prepare(`
SELECT DISTINCT p.* FROM photos p
JOIN photo_hashes ph ON p.id = ph.photo_id
JOIN image_hashes ih ON ph.hash_id = ih.id
WHERE ih.file_count > 1
ORDER BY p.created_at DESC
`)
return selectPhotos.all() as Photo[]
}
// Thumbnail caching operations
getCachedThumbnail(photoId: string, size: number): Buffer | null {
const selectThumbnail = this.db.prepare(`
SELECT thumbnail_blob FROM photos
WHERE id = ? AND thumbnail_size = ? AND thumbnail_blob IS NOT NULL
`)
const result = selectThumbnail.get(photoId, size) as { thumbnail_blob: Buffer } | null
return result?.thumbnail_blob || null
}
updateThumbnailCache(photoId: string, size: number, thumbnailData: Buffer): boolean {
const updateThumbnail = this.db.prepare(`
UPDATE photos
SET thumbnail_blob = ?, thumbnail_size = ?, thumbnail_generated_at = CURRENT_TIMESTAMP
WHERE id = ?
`)
const result = updateThumbnail.run(thumbnailData, size, photoId)
return result.changes > 0
}
clearThumbnailCache(photoId?: string): boolean {
if (photoId) {
const clearThumbnail = this.db.prepare(`
UPDATE photos
SET thumbnail_blob = NULL, thumbnail_size = NULL, thumbnail_generated_at = NULL
WHERE id = ?
`)
const result = clearThumbnail.run(photoId)
return result.changes > 0
} else {
// Clear all thumbnail caches
const clearAllThumbnails = this.db.prepare(`
UPDATE photos
SET thumbnail_blob = NULL, thumbnail_size = NULL, thumbnail_generated_at = NULL
`)
const result = clearAllThumbnails.run()
return result.changes > 0
}
}
// Conflict detection and handling methods
checkForPhotoConflict(filepath: string, sha256Hash: string): { hasConflict: boolean, existingPhoto?: Photo, isDuplicate: boolean } {
const existingPhoto = this.getPhotoByPath(filepath)
if (!existingPhoto) {
return { hasConflict: false, isDuplicate: false }
}
// Get the hash of the existing photo
const getPhotoHash = this.db.prepare(`
SELECT ih.sha256_hash
FROM photo_hashes ph
JOIN image_hashes ih ON ph.hash_id = ih.id
WHERE ph.photo_id = ?
`)
const existingHashRow = getPhotoHash.get(existingPhoto.id) as { sha256_hash: string } | null
if (!existingHashRow) {
// Existing photo has no hash, assume it's different content
return { hasConflict: true, existingPhoto, isDuplicate: false }
}
const existingHash = existingHashRow.sha256_hash
if (existingHash === sha256Hash) {
// Same file, same content - it's a duplicate
return { hasConflict: false, existingPhoto, isDuplicate: true }
} else {
// Same path, different content - it's a conflict
return { hasConflict: true, existingPhoto, isDuplicate: false }
}
}
createPhotoConflict(
photoData: Omit<Photo, 'id' | 'indexed_at' | 'thumbnail_blob' | 'thumbnail_size' | 'thumbnail_generated_at'>,
originalPhotoId: string,
conflictReason: string
): string {
const id = randomUUID()
const insertConflict = this.db.prepare(`
INSERT INTO photo_conflicts (
id, filename, filepath, directory, filesize, created_at, modified_at,
width, height, format, favorite, rating, description, metadata,
original_photo_id, conflict_reason
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`)
try {
const params = [
id, // string
photoData.filename, // string
photoData.filepath, // string
photoData.directory, // string
photoData.filesize, // number
photoData.created_at, // string
photoData.modified_at, // string
photoData.width || null, // number or null
photoData.height || null, // number or null
photoData.format || null, // string or null
photoData.favorite || false, // boolean
photoData.rating || null, // number or null
photoData.description || null, // string or null
photoData.metadata || null, // string (JSON) or null
originalPhotoId, // string
conflictReason // string
]
insertConflict.run(...params)
return id
} catch (error) {
console.error('Error creating photo conflict:', error)
throw new Error('Failed to create photo conflict record')
}
}
getPhotoConflicts(): Array<Photo & { original_photo_id: string, conflict_reason: string, conflict_detected_at: string }> {
const selectConflicts = this.db.prepare('SELECT * FROM photo_conflicts ORDER BY conflict_detected_at DESC')
return selectConflicts.all() as Array<Photo & { original_photo_id: string, conflict_reason: string, conflict_detected_at: string }>
}
// Auto-tagging methods
createOrGetTag(tagName: string, color?: string): Tag {
// Try to get existing tag
const selectTag = this.db.prepare('SELECT * FROM tags WHERE name = ?')
let tag = selectTag.get(tagName.toLowerCase()) as Tag | null
if (tag) {
return tag
}
// Create new tag
const id = randomUUID()
const insertTag = this.db.prepare(`
INSERT INTO tags (id, name, color) VALUES (?, ?, ?)
`)
insertTag.run(id, tagName.toLowerCase(), color || '#3B82F6')
return {
id,
name: tagName.toLowerCase(),
color: color || '#3B82F6',
created_at: new Date().toISOString()
}
}
addPhotoTag(photoId: string, tagName: string, confidence?: number): boolean {
const tag = this.createOrGetTag(tagName)
// Check if association already exists
const existingAssociation = this.db.prepare(
'SELECT * FROM photo_tags WHERE photo_id = ? AND tag_id = ?'
).get(photoId, tag.id)
if (existingAssociation) {
return false // Already exists
}
// Create association
const insertPhotoTag = this.db.prepare(`
INSERT INTO photo_tags (photo_id, tag_id, added_at) VALUES (?, ?, ?)
`)
try {
insertPhotoTag.run(photoId, tag.id, new Date().toISOString())
return true
} catch (error) {
console.error('Error adding photo tag:', error)
return false
}
}
addPhotoTags(photoId: string, tags: Array<{ name: string, confidence?: number }>): number {
let addedCount = 0
for (const tagInfo of tags) {
if (this.addPhotoTag(photoId, tagInfo.name, tagInfo.confidence)) {
addedCount++
}
}
return addedCount
}
getPhotoTags(photoId: string): Tag[] {
const selectTags = this.db.prepare(`
SELECT t.* FROM tags t
INNER JOIN photo_tags pt ON t.id = pt.tag_id
WHERE pt.photo_id = ?
ORDER BY t.name
`)
return selectTags.all(photoId) as Tag[]
}
removePhotoTag(photoId: string, tagId: string): boolean {
const deletePhotoTag = this.db.prepare('DELETE FROM photo_tags WHERE photo_id = ? AND tag_id = ?')
const result = deletePhotoTag.run(photoId, tagId)
return result.changes > 0
}
clearPhotoTags(photoId: string): number {
const deleteAllPhotoTags = this.db.prepare('DELETE FROM photo_tags WHERE photo_id = ?')
const result = deleteAllPhotoTags.run(photoId)
return result.changes
}
searchPhotosByTags(tagNames: string[]): Photo[] {
if (tagNames.length === 0) return []
const placeholders = tagNames.map(() => '?').join(',')
const query = `
SELECT DISTINCT p.* FROM photos p
INNER JOIN photo_tags pt ON p.id = pt.photo_id
INNER JOIN tags t ON pt.tag_id = t.id
WHERE t.name IN (${placeholders})
ORDER BY p.created_at DESC
`
const selectPhotos = this.db.prepare(query)
return selectPhotos.all(...tagNames.map(name => name.toLowerCase())) as Photo[]
}
}
export const photoService = new PhotoService()