edit_image.py

import os
import argparse
from PIL import Image
import json
from moviepy.editor import VideoFileClip
import numpy as np

import black_box_image_edit as image_edit

def infer_video(model, video_path, output_dir, prompt, prompt_type="instruct", force_512=False, seed=42, negative_prompt="", overwrite=False):
    """
    Processes videos from the input directory, resizes them to 512x512 before feeding into the model by first frame,
    and saves the processed video back to its original size in the output directory.

    Args:
        model: The video editing model.
        input_dir (str): Path to the directory containing input videos.
        output_dir (str): Path to the directory where processed videos will be saved.
        prompt (str): Instruction prompt for video editing.
    """

    # Create the output directory if it does not exist
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    video_clip = VideoFileClip(video_path)
    video_filename = os.path.basename(video_path)
    # filename_noext = os.path.splitext(video_filename)[0]
    
    # Create the output directory if it does not exist
    # final_output_dir = os.path.join(output_dir, filename_noext)
    final_output_dir = output_dir
    if not os.path.exists(final_output_dir):
        os.makedirs(final_output_dir)

    result_path = os.path.join(final_output_dir, prompt + ".png")

    # Check if result already exists
    if os.path.exists(result_path) and overwrite is False:
        print(f"Result already exists: {result_path}")
        return

    def process_frame(image):
        pil_image = Image.fromarray(image)
        if force_512:
            pil_image = pil_image.resize((512, 512), Image.LANCZOS)
        if prompt_type == "instruct":
            result = model.infer_one_image(pil_image, instruct_prompt=prompt, seed=seed, negative_prompt=negative_prompt)
        else:
            result = model.infer_one_image(pil_image, target_prompt=prompt, seed=seed, negative_prompt=negative_prompt)
        if force_512:
            result = result.resize(video_clip.size, Image.LANCZOS)
        return np.array(result)
    
    # Process only the first frame
    first_frame = video_clip.get_frame(0)  # Get the first frame
    processed_frame = process_frame(first_frame)  # Process the first frame


    #Image.fromarray(first_frame).save(os.path.join(final_output_dir, "00000.png"))
    Image.fromarray(processed_frame).save(result_path)
    print(f"Processed and saved the first frame: {result_path}")
    return result_path


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Process some images.')
    parser.add_argument('--model', type=str, default='instructpix2pix', choices=['magicbrush','instructpix2pix'], help='Name of the image editing model')
    parser.add_argument('--video_path', type=str, required=False, help='Name of the video', default=None)
    parser.add_argument('--input_dir', type=str, required=False, help='Directory containing the video', default="./demo/")
    parser.add_argument('--output_dir', type=str, required=False, help='Directory to save the processed images', default=None)
    parser.add_argument('--prompt', type=str, required=False, help='Instruction prompt for editing', default="turn the man into darth vader")
    parser.add_argument('--force_512', action='store_true', help='Force resize to 512x512 when feeding into image model')
    parser.add_argument('--dict_file', type=str, required=False, help='JSON file containing files, instructions etc.', default=None)
    parser.add_argument('--seed', type=int, required=False, help='Seed for random number generator', default=42)
    parser.add_argument('--negative_prompt', type=str, required=False, help='Negative prompt for editing', default=None)
    args = parser.parse_args()

    if args.negative_prompt is None:
        negative_prompt = "worst quality, normal quality, low quality, low res, blurry, watermark, jpeg artifacts"
    else:
        negative_prompt = args.negative_prompt
        
    if args.dict_file:
        with open(args.dict_file, 'r') as json_file:
            folders_info = json.load(json_file)

        for video_name, video_infos in folders_info.items():
            input_dir = args.input_dir
            video_path = os.path.join(input_dir, video_name)

            for video_info in video_infos:
                model_name = video_info.get('image_model', None)
                instruction = video_info.get('instruction', None)
                target_caption = video_info.get('target_caption', None)

                if instruction is None and target_caption is None:
                    continue

                if model_name == 'magicbrush':
                    model = image_edit.MagicBrush()
                    prompt_type = "instruct"
                    prompt = instruction
                elif model_name == 'instructpix2pix':
                    model = image_edit.InstructPix2Pix()
                    prompt_type = "instruct"
                    prompt = instruction

                if args.output_dir is None:
                    video_filename = os.path.basename(video_path)
                    filename_noext = os.path.splitext(video_filename)[0]
                    output_dir = os.path.dirname(video_path)
                else:
                    output_dir = args.output_dir

                infer_video(model, video_path, output_dir, prompt, prompt_type, args.force_512, args.seed, negative_prompt)
    else:
        if args.model == 'magicbrush':
            model = image_edit.MagicBrush()
            prompt_type = "instruct"
        elif args.model == 'instructpix2pix':
            model = image_edit.InstructPix2Pix()
            prompt_type = "instruct"

        video_path = args.video_path
        
        if args.output_dir is None:
            video_filename = os.path.basename(video_path)
            filename_noext = os.path.splitext(video_filename)[0]
            output_dir = os.path.dirname(video_path)
        else:
            output_dir = args.output_dir
        
        print("video_filename", video_filename)
        print("output_dir", output_dir)

        infer_video(model, video_path, output_dir, args.prompt, prompt_type, args.force_512, args.seed, negative_prompt)