Fix tourview: Add video and image as way to create tourview

app/domains/supabase/service.py

@@ -70,19 +70,15 @@ class SupabaseService:
             raise InternalServerException(
                 f"Failed to delete old image from storage: {e}"
             )
-        # Prepare the new file for upload.
         new_image_content = await file.read()
         file_extension = file.filename.split(".")[-1] if "." in file.filename else "jpg"
-        # Reuse the old filename if available; otherwise, generate a new one.
         filename = old_filename if old_filename else f"{uuid.uuid4()}.{file_extension}"
-        # Write file content to a temporary file.
         fd, tmp_path = tempfile.mkstemp()
         try:
             with open(tmp_path, "wb") as tmp_file:
                 tmp_file.write(new_image_content)
         finally:
             os.close(fd)
-        # Upload the new file to Supabase Storage.
         try:
             self.supabase_client.storage.from_(bucket_name).upload(
                 filename,
@@ -100,6 +96,5 @@ class SupabaseService:
         )
         return public_url
 
-
 def provide_supabase_service(bucket_name: str) -> SupabaseService:
     return SupabaseService(bucket_name=bucket_name)

app/domains/tourview/controller.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
-from typing import List, Union
+from typing import Any, List, Union
 import uuid
-from litestar import Controller, Response, get, post, status_codes
+from litestar import Controller, Request, Response, get, post
 from litestar.di import Provide
 from litestar.params import Body
 from litestar.datastructures import UploadFile
+from database.models.user import User
+from domains.notification.service import NotificationService
 from domains.tourview.service import TourviewService, provide_tourview_service
 from domains.tourview.dtos import (
     StartTransferSessionDTO,
@@ -12,6 +14,8 @@ from domains.tourview.dtos import (
 )
 from database.models.tourview import Tourview
 from litestar.background_tasks import BackgroundTask
+from litestar.security.jwt import Token
+
 
 class TourviewController(Controller):
     path = "/properties/{property_id:uuid}/tourview"
@@ -58,7 +62,7 @@ class TourviewController(Controller):
     @post(
         "/transfer/{session_id:uuid}/chunk",
         path_override="/tourview/transfer/{session_id:uuid}/chunk",  # Override class path
-        no_auth=True,
+        # no_auth=True,
     )
     async def upload_chunk(
         self,
@@ -76,24 +80,60 @@ class TourviewController(Controller):
     @post(
         "/transfer/{session_id:uuid}/finalize",
         # path_override="/tourview/transfer/{session_id:uuid}/finalize",  # Override class path
-        no_auth=True,
+        # no_auth=True,
     )
     async def finalize_transfer(
         self,
         property_id: uuid.UUID,
+        request: Request[User, Token, Any],
         service: TourviewService,
         session_id: uuid.UUID,
-        
     ) -> str:
         """
         Extra Endpoint: Finalizes a chunked transfer after all chunks are uploaded.
         This is necessary to trigger the processing of video/panorama files.
         """
         result, name = await service.finalize_transfer(session_id)
-        background_task = BackgroundTask(self.create_tourview, result, property_id, name, service)
+        background_task = BackgroundTask(
+            self.create_tourview, request.user, result, property_id, name, service
+        )
         return Response("Transfer completed successfully.", background=background_task)
-    async def create_tourview(self, path: Union[str, list[str]], property_id: uuid.UUID, name: str, service: TourviewService):
-        if path is str:
-            service.stitch_video(path, property_id, name)
+
+    async def notify_partner(self, user: User, tourview: Union[Tourview, None]):
+        if not user.device_token:
+            return
+        notify_service = NotificationService()
+        title = "Tourview Fail" if tourview is None else "Tourview Complete"
+        body = (
+            f"Fail when trying to create your tourview."
+            if tourview is None
+            else f"Your tourview for {tourview.name} is complete. Try it right away!!!"
+        )
+        notify_service.send_to_token(
+            token=user.device_token,
+            title=title,
+            body=body,
+            data=(
+                {
+                    "type": "tourview",
+                    "id": str(tourview.id),
+                    "url": str(tourview.image.url),
+                }
+                if tourview
+                else None
+            ),
+        )
+
+    async def create_tourview(
+        self,
+        user: User,
+        path: Union[str, list[str]],
+        property_id: uuid.UUID,
+        name: str,
+        service: TourviewService,
+    ):
+        if isinstance(path, str):
+            tourview = await service.stitch_video(path, property_id, name)
         else:
-            service.stitch_images(path, property_id, name)
+            tourview = await service.stitch_images(path, property_id, name)
+        await self.notify_partner(user, tourview)

app/domains/tourview/service.py

@@ -1,4 +1,7 @@
 from collections.abc import AsyncGenerator
+from io import BytesIO
+import os
+from PIL import Image as PILImage
 from pathlib import Path
 import shutil
 import cv2
@@ -6,13 +9,17 @@ from typing import List, Optional, Tuple, Union
 import uuid
 from advanced_alchemy.repository import SQLAlchemyAsyncRepository
 from advanced_alchemy.service import SQLAlchemyAsyncRepositoryService
-import numpy as np
+from database.models.image import Image
+from domains.image.service import ImageService
+from domains.tourview.utils import (
+    generate_panorama_image_from_path,
+    generate_panorama_image_from_video,
+)
 from domains.tourview.dtos import (
     StartTransferSessionDTO,
 )
 from database.models.tourview import Tourview
 from sqlalchemy.ext.asyncio import AsyncSession
-from PIL import Image
 from domains.supabase.service import SupabaseService
 import aiofiles
 
@@ -81,7 +88,7 @@ class TourviewService(SQLAlchemyAsyncRepositoryService[Tourview]):
 
         # 3. Create database records
         # NOTE: final_image_path should be a public URL after moving it to a public storage (e.g., S3)
-        new_image = Image(
+        new_image = PILImage(
             url=str(final_image_path),  # This should be a URL, not a local path in prod
             model_id=prop.id,
             model_type="property_tourview",
@@ -157,7 +164,7 @@ class TourviewService(SQLAlchemyAsyncRepositoryService[Tourview]):
                 final_processed_path = await self.reassemble_and_process_chunks(
                     file_paths, session["type"]
                 )
-                return final_processed_path, session["name"]
+                return str(final_processed_path), session["name"]
             elif session["type"] == "image":
                 return file_paths, session["name"]
             else:
@@ -192,54 +199,101 @@ class TourviewService(SQLAlchemyAsyncRepositoryService[Tourview]):
 
         print(f"Reassembly complete. Final file at {final_file}")
         return final_file
-
-    def extract_and_select_frames(
-        self, video_path, min_movement=5, max_movement=50, step=5
-    ):
-        cap = cv2.VideoCapture(video_path)
-        if not cap.isOpened():
-            raise RuntimeError("Could not open video.")
-
-        selected_frames = []
-        prev_gray = None
-        frame_idx = 0
-
-        while True:
-            ret, frame = cap.read()
-            if not ret:
-                break
-
-            if frame_idx % step != 0:
-                frame_idx += 1
-                continue
-
-            gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
-            if prev_gray is not None:
-                # Calculate optical flow or feature movement
-                flow = cv2.calcOpticalFlowFarneback(
-                    prev_gray, gray, None, 0.5, 3, 15, 3, 5, 1.2, 0
+    
+    async def stitch_video(
+        self, path: str, property_id: uuid.UUID, name: str
+    ) -> Tourview:
+        try:
+            panorama = generate_panorama_image_from_video(path)
+            image_rgb = cv2.cvtColor(panorama, cv2.COLOR_BGR2RGB)
+            pil_image = PILImage.fromarray(image_rgb)
+            buffer = BytesIO()
+            pil_image.save(buffer, format="PNG")
+            buffer.seek(0)
+            bucket = self.supabase_service.supabase_client.storage.from_("tourview")
+            try:
+                result = bucket.upload(
+                    f"tourview_{property_id}_{name}.png",
+                    buffer.read(),
+                    {"content-type": "image/png"},
                 )
-                movement = np.linalg.norm(flow, axis=2).mean()
-                print(f"Frame {frame_idx}: avg movement={movement:.2f}")
-
-                if min_movement < movement < max_movement:
-                    selected_frames.append(frame)
-            else:
-                # Always select the first frame
-                selected_frames.append(frame)
-
-            prev_gray = gray
-            frame_idx += 1
-
-        cap.release()
-        return selected_frames
-
-    def stitch_video(self, path: str, property_id: uuid.UUID, name: str) -> Tourview:
-        pass
-
-    def stitch_images(self, paths: list[str], property_id: uuid.UUID, name: str) -> Tourview:
-        pass
+            except Exception as e:
+                raise InternalServerException("Unable to connect to Storage Service")
+            try:
+                public_url = bucket.get_public_url(f"tourview_{property_id}_{name}.png")
+            except:
+                raise InternalServerException(
+                    f"Unable to find the public url for tourview_{property_id}_{name}.png"
+                )
+            image_service = ImageService(session=self.repository.session)
+            image = await image_service.create(
+                Image(
+                    **{
+                        "url": public_url,
+                        "model_id": None,
+                        "model_type": None,
+                    }
+                )
+            )
+            tourview = await self.create(
+                Tourview(image_id=image.id, property_id=property_id, name=name)
+            )
+            return tourview
+        except Exception as e:
+            print(e)
+            await self.repository.session.rollback()
+            raise InternalServerException("Error when trying to create your tourview")
+        finally:
+            await self.repository.session.commit()
+            os.remove(path)
 
+    async def stitch_images(
+        self, paths: list[str], property_id: uuid.UUID, name: str
+    ) -> Tourview:
+        try:
+            panorama = generate_panorama_image_from_path(paths)
+            image_rgb = cv2.cvtColor(panorama, cv2.COLOR_BGR2RGB)
+            pil_image = PILImage.fromarray(image_rgb)
+            buffer = BytesIO()
+            pil_image.save(buffer, format="PNG")
+            buffer.seek(0)
+            bucket = self.supabase_service.supabase_client.storage.from_("tourview")
+            try:
+                result = await bucket.upload(
+                    f"tourview_{property_id}_{name}.png",
+                    buffer.read(),
+                    {"content-type": "image/png"},
+                )
+            except Exception as e:
+                raise InternalServerException("Unable to connect to Storage Service")
+            try:
+                public_url = bucket.get_public_url(f"tourview_{property_id}_{name}.png")
+            except:
+                raise InternalServerException(
+                    f"Unable to find the public url for tourview_{property_id}_{name}.png"
+                )
+            image_service = ImageService(session=self.repository.session)
+            image = await image_service.create(
+                Image(
+                    **{
+                        "url": public_url,
+                        "model_id": None,
+                        "model_type": None,
+                    }
+                )
+            )
+            tourview = await self.create(
+                Tourview(image_id=image.id, property_id=property_id, name=name)
+            )
+            return tourview
+        except Exception as e:
+            print(e)
+            await self.repository.session.rollback()
+            raise InternalServerException("Error when trying to create your tourview")
+        finally:
+            await self.repository.session.commit()
+            for path in paths:
+                os.remove(path)
 
 async def provide_tourview_service(
     db_session: AsyncSession,

app/domains/tourview/utils.py

@@ -0,0 +1,186 @@
+from typing import List
+import torch
+from lightglue.utils import rbd
+from lightglue import LightGlue, SuperPoint
+import networkx as nx
+from tqdm import tqdm
+import gc
+device = "cuda" if torch.cuda.is_available() else "cpu"
+from PIL import Image
+from torchvision import transforms as t
+
+tf = t.Compose(
+    [
+        t.ToTensor(),  # converts to (C, H, W), [0,1]
+    ]
+)
+import cv2
+import numpy as np
+
+import cv2
+from tqdm import tqdm
+
+
+def pad_to_equirectangular(image):
+    h, w = image.shape[:2]
+    target_height = w // 2
+
+    if h == target_height:
+        print("Image is already 2:1 equirectangular.")
+        return image
+    elif h > target_height:
+        print(f"Resizing image from ({w}x{h}) to ({w}x{target_height}) to match 2:1 ratio.")
+        resized_image = cv2.resize(image, (w, target_height), interpolation=cv2.INTER_AREA)
+        return resized_image
+    pad_total = target_height - h
+    print(f"Padding: {pad_total} px to reach {target_height}px height.")
+    padded_image = cv2.copyMakeBorder(
+        image,
+        pad_total,
+        0,
+        0,
+        0,
+        borderType=cv2.BORDER_CONSTANT,
+        value=(0, 0, 0),  # black padding
+    )
+
+    return padded_image
+
+
+def read_images(image_paths):
+    images = []
+    for path in image_paths:
+        img = cv2.imread(path)
+        if img is None:
+            raise ValueError(f"Could not read {path}")
+        images.append(img)
+    return images
+
+
+def compute_matches(images, match_conf=0.6):
+    extractor = SuperPoint(max_num_keypoints=512).eval().to(device)
+    matcher = LightGlue(features="superpoint").eval().to(device)
+    features, feats_np = [], []
+    for img in images:
+        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+        gray = Image.fromarray(gray, mode="L")
+        gray = tf(gray).unsqueeze(0).to(device)
+        feat = extractor.extract(gray, device=device)
+        features.append(feat)
+        feats_np.append(gray)
+    G = nx.Graph()
+    for i in range(len(images)):
+        G.add_node(i)  # no need for image_paths
+    print("Computing pairwise matches using LightGlue with progress bar...")
+    pairs = [(i, j) for i in range(len(images)) for j in range(i + 1, len(images))]
+    for i, j in tqdm(pairs, desc="Matching pairs", unit="pair"):
+        f0, f1 = features[i], features[j]
+        match_dict = matcher({"image0": f0, "image1": f1})
+        f0, f1, match_dict = [rbd(x) for x in [f0, f1, match_dict]]
+        matches = match_dict["matches"].cpu().numpy()  # (K,2)
+        if matches.shape[0] < 5:
+            continue
+        kpts0 = f0["keypoints"][matches[:, 0]].cpu().numpy()
+        kpts1 = f1["keypoints"][matches[:, 1]].cpu().numpy()
+        if len(kpts1) < 8:
+            continue
+        H, inliers = cv2.findHomography(kpts0, kpts1, cv2.RANSAC, 4.0)
+        inlier_ratio = np.sum(inliers) / len(inliers) if inliers is not None else 0
+        if H is not None and inlier_ratio > match_conf:
+            conf = inlier_ratio
+            G.add_edge(i, j, weight=-conf)
+            tqdm.write(f"Match {i}-{j}: {len(kpts1)} matches, inlier ratio={conf:.2f}")
+        del match_dict, f0, f1, matches
+    del features, feats_np, matcher, extractor
+    return G
+
+def compute_mst_order(G, start):
+    mst = nx.minimum_spanning_tree(G, weight="weight")
+    order = list(nx.dfs_preorder_nodes(mst, source=start))
+    print("MST-based order:", order)
+    return order
+
+
+def generate_panorama_images(images):
+    if len(images) < 2:
+        print("Need at least two images to stitch a panorama.")
+        return None
+    stitcher = cv2.Stitcher_create(cv2.Stitcher_PANORAMA)
+    try:
+        cv2.ocl.setUseOpenCL(False)
+        stitcher.setWaveCorrection(True)
+        stitcher.setRegistrationResol(0.6)
+        stitcher.setSeamEstimationResol(0.6)
+        stitcher.setInterpolationFlags(cv2.INTER_CUBIC)
+    except Exception as e:
+        print(f"Warning: could not set advanced options: {e}")
+
+    status, pano = stitcher.stitch(images)
+    if status != cv2.Stitcher_OK:
+        print(f"Error during stitching: {status}")
+        return None
+    return pano
+
+def fill_black_with_inpainting(image):
+    mask = np.all(image == 0, axis=2).astype(np.uint8) * 255
+    impainted = cv2.inpaint(image, mask, inpaintRadius=5, flags=cv2.INPAINT_TELEA)
+    return impainted
+
+def generate_panorama_image_from_path(image_paths: List[str]):
+    images = read_images(image_paths)
+    graph = compute_matches(images, match_conf=0.6)
+    degrees = dict(graph.degree())
+    start = max(degrees, key=degrees.get)
+    mst_order = compute_mst_order(graph, start)
+    panorama = generate_panorama_images([images[i] for i in mst_order])
+    if panorama is None:
+        return None
+    panorama = fill_black_with_inpainting(panorama)
+    panorama = pad_to_equirectangular(panorama)
+    return fill_black_with_inpainting(panorama)
+
+def extract_and_select_frames(video_path, min_movement=3, max_movement=50, step=3):
+    cap = cv2.VideoCapture(video_path)
+    if not cap.isOpened():
+        raise RuntimeError("Could not open video.")
+
+    selected_frames = []
+    prev_gray = None
+    frame_idx = 0
+
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        if frame_idx % step != 0:
+            frame_idx += 1
+            continue
+
+        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+        if prev_gray is not None:
+            flow = cv2.calcOpticalFlowFarneback(prev_gray, gray, None, 0.3, 3, 15, 3, 8, 1.5, 0)
+            movement = np.linalg.norm(flow, axis=2).mean()
+            print(f"Frame {frame_idx}: avg movement={movement:.2f}")
+            if min_movement < movement < max_movement:
+                selected_frames.append(frame)
+        else:
+            selected_frames.append(frame)
+
+        prev_gray = gray
+        frame_idx += 1
+    cap.release()
+    return selected_frames
+def generate_panorama_image_from_video(video_path: str):
+    images = extract_and_select_frames(video_path)
+    gc.collect()
+    graph = compute_matches(images, match_conf=0.6)
+    degrees = dict(graph.degree())
+    start = max(degrees, key=degrees.get)
+    mst_order = compute_mst_order(graph, start)
+    panorama = generate_panorama_images([images[i] for i in mst_order])
+    if panorama is None:
+        return None
+    panorama = fill_black_with_inpainting(panorama)
+    panorama = pad_to_equirectangular(panorama)
+    return fill_black_with_inpainting(panorama)

app/migrations/versions/2025-07-07_tourvie2w_b4291c447067.py

@@ -0,0 +1,82 @@
+# type: ignore
+"""tourvie2w
+
+Revision ID: b4291c447067
+Revises: 97836c3c7d1b
+Create Date: 2025-07-07 12:22:14.516779
+
+"""
+
+import warnings
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+from advanced_alchemy.types import EncryptedString, EncryptedText, GUID, ORA_JSONB, DateTimeUTC
+from sqlalchemy import Text  # noqa: F401
+from sqlalchemy.dialects import postgresql
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+__all__ = ["downgrade", "upgrade", "schema_upgrades", "schema_downgrades", "data_upgrades", "data_downgrades"]
+
+sa.GUID = GUID
+sa.DateTimeUTC = DateTimeUTC
+sa.ORA_JSONB = ORA_JSONB
+sa.EncryptedString = EncryptedString
+sa.EncryptedText = EncryptedText
+
+# revision identifiers, used by Alembic.
+revision = 'b4291c447067'
+down_revision = '97836c3c7d1b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", category=UserWarning)
+        with op.get_context().autocommit_block():
+            schema_upgrades()
+            data_upgrades()
+
+def downgrade() -> None:
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", category=UserWarning)
+        with op.get_context().autocommit_block():
+            data_downgrades()
+            schema_downgrades()
+
+def schema_upgrades() -> None:
+    """schema upgrade migrations go here."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('tourviews',
+    sa.Column('id', sa.UUID(), nullable=False),
+    sa.Column('name', sa.String(length=255), nullable=False),
+    sa.Column('image_id', sa.UUID(), nullable=False),
+    sa.Column('property_id', sa.UUID(), nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
+    sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
+    sa.ForeignKeyConstraint(['image_id'], ['images.id'], name=op.f('fk_tourviews_image_id_images'), ondelete='SET NULL'),
+    sa.ForeignKeyConstraint(['property_id'], ['properties.id'], name=op.f('fk_tourviews_property_id_properties'), ondelete='SET NULL'),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_tourviews')),
+    sa.UniqueConstraint('id'),
+    sa.UniqueConstraint('id', name=op.f('uq_tourviews_id')),
+    sa.UniqueConstraint('image_id'),
+    sa.UniqueConstraint('image_id', name=op.f('uq_tourviews_image_id')),
+    sa.UniqueConstraint('property_id'),
+    sa.UniqueConstraint('property_id', name=op.f('uq_tourviews_property_id'))
+    )
+    # ### end Alembic commands ###
+
+def schema_downgrades() -> None:
+    """schema downgrade migrations go here."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('tourviews')
+    # ### end Alembic commands ###
+
+def data_upgrades() -> None:
+    """Add any optional data upgrade migrations here!"""
+
+def data_downgrades() -> None:
+    """Add any optional data downgrade migrations here!"""