Spiiin's blog

import os
import xai_sdk
import base64

client = xai_sdk.Client(api_key="КЛЮЧ")

with open("wizzard.png", "rb") as f:        # или .jpg, .jpeg и т.д.
    image_bytes = f.read()
    image_base64 = base64.b64encode(image_bytes).decode("utf-8")
image_data_uri = f"data:image/png;base64,{image_base64}"
    
response = client.video.generate(
    image_url=image_data_uri,
    prompt="волшебник атакует посохом, колдует магию, атака направлена вправо. no vfx",
    model="grok-imagine-video",
    duration=2,
    aspect_ratio="4:3",
    resolution="480p",
)
print(response.url)
# забираем mp4 файл

import os
import OpenEXR
import Imath
import numpy as np
from PIL import Image

INPUT_DIR = "."
OUTPUT_DIR = "png_out"

os.makedirs(OUTPUT_DIR, exist_ok=True)


def read_exr(exr_path):
    exr_file = OpenEXR.InputFile(exr_path)
    header = exr_file.header()
    dw = header["dataWindow"]

    width = dw.max.x - dw.min.x + 1
    height = dw.max.y - dw.min.y + 1

    channels = header["channels"].keys()
    pt = Imath.PixelType(Imath.PixelType.FLOAT)

    def read_channel(name):
        return np.frombuffer(
            exr_file.channel(name, pt),
            dtype=np.float32
        ).reshape((height, width))

    r = read_channel("R") if "R" in channels else None
    g = read_channel("G") if "G" in channels else None
    b = read_channel("B") if "B" in channels else None
    a = read_channel("A") if "A" in channels else None

    if r is None:
        raise RuntimeError("No RGB channels found")

    return r, g, b, a


def linear_to_srgb(x):
    return np.where(
        x <= 0.0031308,
        12.92 * x,
        1.055 * np.power(np.clip(x, 0.0, None), 1.0 / 2.4) - 0.055
    )


def exr_to_png(src_path, dst_path):
    r, g, b, a = read_exr(src_path)

    rgb = np.stack([r, g, b], axis=-1)

    rgb = np.nan_to_num(rgb, nan=0.0, posinf=1.0, neginf=0.0)
    rgb = np.clip(rgb, 0.0, 1.0)

    rgb = linear_to_srgb(rgb)

    rgb8 = (rgb * 255).astype(np.uint8)

    if a is not None:
        alpha = np.nan_to_num(a, nan=0.0)
        alpha = np.clip(alpha, 0.0, 1.0)
        alpha8 = (alpha * 255).astype(np.uint8)

        rgba = np.dstack([rgb8, alpha8])
        img = Image.fromarray(rgba, mode="RGBA")

    else:
        img = Image.fromarray(rgb8, mode="RGB")

    img.save(dst_path)


def main():
    for filename in os.listdir(INPUT_DIR):
        if filename.lower().endswith(".exr"):
            src = os.path.join(INPUT_DIR, filename)
            dst = os.path.join(
                OUTPUT_DIR,
                os.path.splitext(filename)[0] + ".png"
            )

            try:
                exr_to_png(src, dst)
                print(f"Converted: {filename}")
            except Exception as e:
                print(f"Failed: {filename} ({e})")


if __name__ == "__main__":
    main()

image_path = f"frame_000001.png"
image = Image.open(image_path).convert("RGB")
width, height = image.size
processor = Sam3Processor(model, confidence_threshold=0.5)
inference_state = processor.set_image(image)

processor.reset_all_prompts(inference_state)
inference_state = processor.set_text_prompt(state=inference_state, prompt="robe and belt")

img0 = Image.open(image_path)
plot_results(img0, inference_state)

import os
from pathlib import Path
import numpy as np
from PIL import Image

import torch
import sam3

from sam3 import build_sam3_image_model
from sam3.train.data.collator import collate_fn_api as collate
from sam3.model.utils.misc import copy_data_to_device
from sam3.train.transforms.basic_for_api import (
    ComposeAPI,
    RandomResizeAPI,
    ToTensorAPI,
    NormalizeAPI,
)
from sam3.eval.postprocessors import PostProcessImage
from sam3.train.data.sam3_image_dataset import (
    InferenceMetadata,
    FindQueryLoaded,
    Image as SAMImage,
    Datapoint,
)

# --------------------------------------------------
# Настройки
# --------------------------------------------------

image_paths = [
    "frame_000021.png",
    "frame_000022.png",
    "frame_000023.png",
    "frame_000024.png",
    "frame_000025.png",
]

prompts = [
    "robe",
    "belt",
    "head",
    "hat",
    "hand",
    "shoe",
    "wooden staff",
    "ball",
    "bag"
]

out_dir = "parts_out"
os.makedirs(out_dir, exist_ok=True)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

transform = ComposeAPI(
    transforms=[
        RandomResizeAPI(
            sizes=1008,
            max_size=1008,
            square=True,
            consistent_transform=False,
        ),
        ToTensorAPI(),
        NormalizeAPI(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
    ]
)

postprocessor = PostProcessImage(
    max_dets_per_img=-1,
    iou_type="segm",
    use_original_sizes_box=True,
    use_original_sizes_mask=True,
    convert_mask_to_rle=False,
    detection_threshold=0.5,
    to_cpu=False,
)

GLOBAL_COUNTER = 1

def create_empty_datapoint():
    return Datapoint(find_queries=[], images=[])


def set_image_to_datapoint(datapoint, pil_image):
    w, h = pil_image.size
    datapoint.images = [SAMImage(data=pil_image, objects=[], size=[h, w])]


def add_text_prompt(datapoint, text_query):
    global GLOBAL_COUNTER
    assert len(datapoint.images) == 1, "Please set image first"

    h, w = datapoint.images[0].size

    query_id = GLOBAL_COUNTER

    datapoint.find_queries.append(
        FindQueryLoaded(
            query_text=text_query,
            image_id=0,
            object_ids_output=[],
            is_exhaustive=True,
            query_processing_order=0,
            inference_metadata=InferenceMetadata(
                coco_image_id=query_id,
                original_image_id=query_id,
                original_category_id=1,
                original_size=[h, w],
                object_id=0,
                frame_index=0,
            ),
        )
    )

    GLOBAL_COUNTER += 1
    return query_id


# --------------------------------------------------
# Формирование batched datapoints
# --------------------------------------------------

# Будем хранить:
# 1) transformed datapoints для collate
# 2) маппинг query_id -> (image_path, prompt)
# 3) оригинальные RGBA изображения для сохранения масок
datapoints = []
query_map = {}
rgba_images = {}

for image_path in image_paths:
    print(f"Prepare datapoint: {image_path}")
    img_rgb = Image.open(image_path).convert("RGB")
    img_rgba = Image.open(image_path).convert("RGBA")
    rgba_images[image_path] = img_rgba

    dp = create_empty_datapoint()
    set_image_to_datapoint(dp, img_rgb)

    for prompt in prompts:
        query_id = add_text_prompt(dp, prompt)
        query_map[query_id] = {
            "image_path": image_path,
            "prompt": prompt,
        }

    dp = transform(dp)
    datapoints.append(dp)

# --------------------------------------------------
# Один batched forward на весь список изображений
# --------------------------------------------------

with torch.inference_mode():
    batch = collate(datapoints, dict_key="dummy")["dummy"]
    batch = copy_data_to_device(batch, device, non_blocking=True)

    # В официальном batched guide inference делается именно так.
    output = model(batch)
    processed_results = postprocessor.process_results(output, batch.find_metadatas)

# --------------------------------------------------
# Сохранение результатов
# --------------------------------------------------

saved_count = 0

for query_id, meta in query_map.items():
    image_path = meta["image_path"]
    prompt = meta["prompt"]

    if query_id not in processed_results:
        print(f"Missing result for query_id={query_id} | {image_path} | {prompt}")
        continue

    result = processed_results[query_id]

    # Обычно тут есть masks / boxes / scores как и в plot_results-примерах.
    masks = result.get("masks", None)
    scores = result.get("scores", None)

    if masks is None or len(masks) == 0:
        print(f"No detections: {image_path} | {prompt}")
        continue

    base = sanitize_filename(prompt)
    stem = Path(image_path).stem
    image_rgba = rgba_images[image_path]

    # Сохраняем все найденные маски по данному prompt
    for i in range(len(masks)):
        out_path = os.path.join(out_dir, f"{stem}_{base}_{i}.png")
        ok = save_masked_part_fullsize(
            image_rgba=image_rgba,
            mask_tensor=masks[i],
            out_path=out_path,
            draw_bbox=False,
            premultiply_rgb=True,
            dilate_pixels=0
        )

        if ok:
            score_text = ""
            if scores is not None and len(scores) > i:
                try:
                    score_text = f" | score={float(scores[i]):.3f}"
                except Exception:
                    score_text = ""

            print(f"Saved {image_path} | {prompt}[{i}] -> {out_path}{score_text}")
            saved_count += 1

print(f"Done. Saved files: {saved_count}")