Sync from development - prepare for v0.5.0.3

jackify/backend/core/modlist_operations_configuration_cli.py

@@ -121,6 +121,9 @@ class ModlistOperationsConfigurationCLIMixin:
             if debug_mode:
                 cmd.append('--debug')
                 self.logger.info("Adding --debug flag to jackify-engine")
+            if self.context.get('skip_disk_check'):
+                cmd.append('--skip-disk-check')
+                self.logger.info("Adding --skip-disk-check flag to jackify-engine")
 
             original_env_values = {
                 'NEXUS_API_KEY': os.environ.get('NEXUS_API_KEY'),

jackify/backend/handlers/modlist_configuration.py

@@ -50,10 +50,20 @@ class ModlistConfigurationMixin:
             return True
 
     def _execute_configuration_steps(self, status_callback=None, manual_steps_completed=False, skip_manual_for_existing=False):
-        """Run the configuration steps for the selected modlist."""
+        """
+        Runs the actual configuration steps for the selected modlist.
+        Args:
+            status_callback (callable, optional): A function to call with status updates during configuration.
+            manual_steps_completed (bool): If True, skip the manual steps prompt (used for new modlist flow).
+            skip_manual_for_existing (bool): If True, always skip manual steps (for existing modlists that are already configured).
+        """
         try:
+            # Store status_callback for Configuration Summary
             self._current_status_callback = status_callback
+            
             self.logger.info("Executing configuration steps...")
+            
+            # Ensure required context is set
             if not all([self.modlist_dir, self.appid, self.game_var, self.steamdeck is not None]):
                 self.logger.error("Cannot execute configuration steps: Missing required context (modlist_dir, appid, game_var, steamdeck status).")
                 self.logger.error("Missing required information to start configuration.")
@@ -79,10 +89,14 @@ class ModlistConfigurationMixin:
             return False # Abort on failure
         self.logger.info("Step 1: Setting Protontricks permissions... Done")
 
+        # Step 2: Prompt user for manual steps and wait for compatdata
         skip_manual_prompt = skip_manual_for_existing  # Existing modlists skip manual steps
         if not manual_steps_completed and not skip_manual_for_existing:
+            # Check if Proton Experimental is already set and compatdata exists
             proton_ok = False
             compatdata_ok = False
+            
+            # Check Proton version
             self.logger.debug(f"[MANUAL STEPS DEBUG] Checking Proton version for AppID {self.appid}")
             if self._detect_proton_version():
                 self.logger.debug(f"[MANUAL STEPS DEBUG] Detected Proton version: {self.proton_ver}")
@@ -92,6 +106,7 @@ class ModlistConfigurationMixin:
             else:
                 self.logger.debug("[MANUAL STEPS DEBUG] Could not detect Proton version")
                 
+            # Check compatdata/prefix
             prefix_path_str = self.path_handler.find_compat_data(str(self.appid))
             self.logger.debug(f"[MANUAL STEPS DEBUG] Compatdata path search result: {prefix_path_str}")
 
@@ -158,6 +173,9 @@ class ModlistConfigurationMixin:
             self.logger.error(f"Failed to download or apply curated user.reg.modlist or system.reg.modlist: {e}")
             self.logger.error(f"Failed to download or apply curated user.reg.modlist or system.reg.modlist. {e}")
             return False
+        self.logger.info("Step 3: Curated user.reg.modlist and system.reg.modlist applied successfully.")
+        # The curated registry files overwrite the entire Wine registry, so any
+        # game-specific entries injected earlier must be re-applied immediately after.
         special_game_type = self.detect_special_game_type(self.modlist_dir)
         if special_game_type in ["fnv", "fo3", "enderal"]:
             self.logger.info(
@@ -166,7 +184,6 @@ class ModlistConfigurationMixin:
             )
             try:
                 from jackify.backend.services.automated_prefix_service import AutomatedPrefixService
-
                 AutomatedPrefixService()._inject_game_registry_entries(prefix_path_str, special_game_type)
             except Exception as e:
                 self.logger.error(
@@ -176,7 +193,6 @@ class ModlistConfigurationMixin:
                 )
                 self.logger.error("Could not restore required game registry entries after applying curated registry files.")
                 return False
-        self.logger.info("Step 3: Curated user.reg.modlist and system.reg.modlist applied successfully.")
 
         # Step 4: Install Wine Components
         if status_callback:
@@ -546,7 +562,9 @@ class ModlistConfigurationMixin:
             status_callback("")  # Blank line after final Prefix Configuration step
         self.logger.info("Step 12: Checking for modlist-specific steps...")
 
-        # Step 13: Launch options for special games are now set during automated workflow
+        # Step 13: Launch options for special games are now set during automated prefix workflow (before Steam restart)
+        # Avoids a second Steam restart
+        special_game_type = self.detect_special_game_type(self.modlist_dir)
         if special_game_type:
             self.logger.info(f"Step 13: Launch options for {special_game_type.upper()} were set during automated workflow")
         else:
@@ -569,12 +587,18 @@ class ModlistConfigurationMixin:
         return True # Return True on success
 
     def run_modlist_configuration_phase(self, context: dict = None) -> bool:
-        """Run the full modlist configuration sequence."""
+        """
+        Main entry point to run the full modlist configuration sequence.
+        This orchestrates all the individual steps.
+        """
         self.logger.info(f"Starting configuration phase for modlist: {self.game_name}")
+        # Call the private method that contains the actual steps
+        # Pass along the status_callback if it was provided in the context
         status_callback = context.get('status_callback') if context else None
         return self._execute_configuration_steps(status_callback=status_callback)
 
     def _prompt_or_set_resolution(self):
+        # If on Steam Deck, set 1280x800 automatically
         if self._is_steam_deck():
             self.selected_resolution = "1280x800"
             self.logger.info("Steam Deck detected: setting resolution to 1280x800.")

jackify/backend/handlers/ttw_installer_backend.py

@@ -91,6 +91,9 @@ class TTWInstallerBackendMixin:
         except Exception as e:
             self.logger.error("Error executing TTW_Linux_Installer: %s", e, exc_info=True)
             return False, f"Error executing TTW_Linux_Installer: {e}"
+        finally:
+            from jackify.shared.paths import cleanup_stale_tmp
+            cleanup_stale_tmp()
 
     def start_ttw_installation(self, ttw_mpi_path: Path, ttw_output_path: Path, output_file: Path):
         """Start TTW installation process (non-blocking). Returns (process, error_message)."""
@@ -168,6 +171,8 @@ class TTWInstallerBackendMixin:
                         process.kill()
                     except Exception:
                         pass
+        from jackify.shared.paths import cleanup_stale_tmp
+        cleanup_stale_tmp()
 
     def install_ttw_backend_with_output_stream(self, ttw_mpi_path: Path, ttw_output_path: Path, output_callback=None):
         """Install TTW with streaming output (DEPRECATED - use start_ttw_installation instead)."""
@@ -251,6 +256,9 @@ class TTWInstallerBackendMixin:
         except Exception as e:
             self.logger.error("Error executing TTW_Linux_Installer: %s", e, exc_info=True)
             return False, f"Error executing TTW_Linux_Installer: {e}"
+        finally:
+            from jackify.shared.paths import cleanup_stale_tmp
+            cleanup_stale_tmp()
 
     @staticmethod
     def integrate_ttw_into_modlist(ttw_output_path: Path, modlist_install_dir: Path, ttw_version: str, skip_copy: bool = False) -> bool:

jackify/backend/services/download_watcher_service.py

@@ -4,6 +4,7 @@ list of pending manual download items by lax filename comparison.
 """
 
 import os
+import re
 import time
 import logging
 from dataclasses import dataclass, field
@@ -106,6 +107,14 @@ class DownloadWatcherService:
                 logger.debug(f"Candidate dot-normalized match: {path.name} -> {expected_name}")
                 self._debounce_and_emit(path, item)
                 return
+        # Some modlist metadata stores filenames with a leading numeric prefix
+        # (e.g. "1_filename.zip") that is absent from the browser-saved file.
+        for expected_name, item in self._pending_exact:
+            stripped = re.sub(r'^\d+_', '', expected_name)
+            if stripped != expected_name and stripped == candidate_name:
+                logger.debug(f"Candidate numeric-prefix match: {path.name} -> {expected_name}")
+                self._debounce_and_emit(path, item)
+                return
 
     def _debounce_and_emit(self, path: Path, item: dict) -> None:
         def _wait_and_emit():

jackify/backend/services/manual_download_manager_runtime_mixin.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import json
 import logging
+import re
 import subprocess
 from pathlib import Path
 from typing import Optional
@@ -380,6 +381,13 @@ class ManualDownloadManagerRuntimeMixin:
                 stripped = name.lower().lstrip('.')
                 if stripped != name.lower():
                     exact = exact_map.get(stripped)
+            if exact is None:
+                # Numeric prefix normalization: engine may store filenames with a
+                # leading numeric prefix (e.g. "1_filename.zip") absent from the
+                # browser-saved file.
+                stripped_num = re.sub(r'^\d+_', '', name.lower())
+                if stripped_num != name.lower():
+                    exact = exact_map.get(stripped_num)
             if exact is None or exact in used_paths:
                 continue
             used_paths.add(exact)

jackify/backend/services/modlist_service_installation.py

@@ -145,6 +145,8 @@ class ModlistServiceInstallationMixin:
             elif context.get('machineid'):
                 cmd += ['-m', context['machineid']]
             cmd += ['-o', install_dir_str, '-d', download_dir_str]
+            if context.get('skip_disk_check'):
+                cmd.append('--skip-disk-check')
 
             original_env_values = {
                 'NEXUS_API_KEY': os.environ.get('NEXUS_API_KEY'),
@@ -199,9 +201,10 @@ class ModlistServiceInstallationMixin:
                     except (OSError, BrokenPipeError):
                         return False
 
-                from jackify.backend.utils.cc_content_detector import is_cc_content_error, extract_cc_filename
+                from jackify.backend.utils.cc_content_detector import is_cc_content_error, extract_cc_filename, is_creation_kit_missing_error
                 import json as _json
                 _cc_filename = None
+                _ck_missing = False
                 _pending_manual: list = []
                 buffer = b''
                 while True:
@@ -263,6 +266,8 @@ class ModlistServiceInstallationMixin:
                             output_callback(decoded)
                         if _cc_filename is None and is_cc_content_error(decoded):
                             _cc_filename = extract_cc_filename(decoded) or ""
+                        if not _ck_missing and is_creation_kit_missing_error(decoded):
+                            _ck_missing = True
 
                 if buffer:
                     line = buffer.decode('utf-8', errors='replace')
@@ -271,6 +276,8 @@ class ModlistServiceInstallationMixin:
                         output_callback(decoded)
                     if _cc_filename is None and is_cc_content_error(decoded):
                         _cc_filename = extract_cc_filename(decoded) or ""
+                    if not _ck_missing and is_creation_kit_missing_error(decoded):
+                        _ck_missing = True
 
                 proc.wait()
                 if proc.returncode != 0:
@@ -285,6 +292,16 @@ class ModlistServiceInstallationMixin:
                         output_callback("  - If specific files are still missing, search for and download them from the Creations menu.")
                         output_callback("  - If problems persist, uninstall and reinstall Skyrim, then launch once to trigger the AE download.")
                         output_callback("  - Note: Skyrim AE via Steam Family Sharing does not transfer DLC content.")
+                    if _ck_missing and output_callback:
+                        output_callback("")
+                        output_callback("[WARN] Creation Kit Files Missing")
+                        output_callback("  This modlist requires the Skyrim Special Edition Creation Kit.")
+                        output_callback("  - In Steam, search for 'Skyrim Special Edition: Creation Kit' and install it.")
+                        output_callback("  - Right-click it in Steam > Properties > Compatibility and set a Proton version.")
+                        output_callback("  - Click Play to launch the Creation Kit.")
+                        output_callback("  - When asked whether to unzip Scripts.zip, select NO.")
+                        output_callback("  - Once the Creation Kit opens successfully, close it.")
+                        output_callback("  - Re-run the modlist install in Jackify.")
                     return False
                 if output_callback:
                     output_callback("Installation completed successfully")

jackify/backend/services/update_service.py

@@ -369,15 +369,70 @@ class UpdateService:
                         if progress_callback:
                             progress_callback(downloaded_size, total_size)
             
+            # Nexus delivers a 7z archive — extract the AppImage before handing off
+            if self._is_7z_archive(temp_file):
+                logger.info("Downloaded file is a 7z archive, extracting AppImage")
+                extracted = self._extract_appimage_from_7z(temp_file, update_dir, update_info.version)
+                temp_file.unlink(missing_ok=True)
+                if not extracted:
+                    logger.error("Failed to extract AppImage from 7z archive")
+                    return None
+                temp_file = extracted
+
             # Make executable
             temp_file.chmod(0o755)
-            
+
             logger.info("Update downloaded successfully: %s from %s -> %s", update_info.version, update_info.source, temp_file)
             return temp_file
-            
+
         except Exception as e:
             logger.error(f"Failed to download update manually: {e}")
             return None
+
+    def _is_7z_archive(self, path: Path) -> bool:
+        """Detect 7z archive by magic bytes (37 7A BC AF 27 1C)."""
+        try:
+            with open(path, 'rb') as f:
+                magic = f.read(6)
+            return magic == b'7z\xbc\xaf\x27\x1c'
+        except Exception:
+            return False
+
+    def _get_bundled_7z_path(self) -> Optional[Path]:
+        """Return path to bundled 7z binary (AppImage or dev)."""
+        import os
+        candidates = []
+        appdir = os.environ.get('APPDIR')
+        if appdir:
+            candidates.append(Path(appdir) / 'opt' / 'jackify' / 'tools' / '7z')
+        candidates.append(Path(__file__).parent.parent.parent / 'tools' / '7z')
+        for p in candidates:
+            if p.exists() and os.access(p, os.X_OK):
+                return p
+        return None
+
+    def _extract_appimage_from_7z(self, archive: Path, dest_dir: Path, version: str) -> Optional[Path]:
+        """Extract Jackify.AppImage from a 7z archive into dest_dir."""
+        seven_z = self._get_bundled_7z_path()
+        if not seven_z:
+            logger.error("Bundled 7z not found, cannot extract update archive")
+            return None
+        out_path = dest_dir / f"Jackify-{version}.AppImage"
+        try:
+            result = subprocess.run(
+                [str(seven_z), 'e', str(archive), 'Jackify.AppImage', f'-o{dest_dir}', '-y'],
+                capture_output=True, text=True, timeout=120
+            )
+            extracted = dest_dir / 'Jackify.AppImage'
+            if result.returncode != 0 or not extracted.exists():
+                logger.error("7z extraction failed (rc=%d): %s", result.returncode, result.stderr.strip())
+                return None
+            extracted.rename(out_path)
+            logger.info("Extracted AppImage from archive: %s", out_path)
+            return out_path
+        except Exception as e:
+            logger.error("Exception during 7z extraction: %s", e)
+            return None
     
     def apply_update(self, new_appimage_path: Path) -> bool:
         """

jackify/backend/utils/cc_content_detector.py

@@ -32,3 +32,28 @@ def extract_cc_filename(line: str) -> Optional[str]:
     """Return the CC filename from a line, or None if not found."""
     m = _CC_FILE_RE.search(line)
     return m.group(0) if m else None
+
+
+# Files that only exist inside the Skyrim SE Creation Kit install.
+# Used to detect modlists that require the CK as a game file source.
+_CK_INDICATORS = (
+    'creationkit',
+    'papyrus compiler',
+    'scriptcompile',
+    'lipgen',
+    'assetwatcher',
+    'havokbehaviorpostprocess',
+    'skyrimreservedaddonindexes',
+    'p4com64',
+    'lex_ssce',
+)
+
+
+def is_creation_kit_missing_error(line: str) -> bool:
+    """Return True if line indicates a missing Creation Kit file (GameFileSource)."""
+    if not line:
+        return False
+    normalized = line.strip().lower()
+    if 'gamefilesource' not in normalized:
+        return False
+    return any(ind in normalized for ind in _CK_INDICATORS)